From a3bde634705c44b042e3cfffaf08079c738b7f61 Mon Sep 17 00:00:00 2001 From: rakesh1002 Date: Thu, 28 May 2026 21:08:54 +0530 Subject: [PATCH] chore: consolidate repo structure; delete stale apps/backend fork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apps/backend/ was a stale fork of root app/ with diverged code: - UnQuestRequest class name (pre-rebrand) vs UnSearchRequest at root - Different import paths (app.services.searxng vs app.services.core.searxng) - 2 Alembic migrations vs 5 at root - pytest.ini pointing at sqlite vs the live root postgres - Independent Dockerfile, docker-compose, requirements.txt, pyproject.toml that nothing live referenced - A 2841-line production SearXNG settings.yml that was never mounted (the live mount path searxng/settings.yml was a 40-line stub falling back to upstream defaults — a real bug) All live references (ci-cd.yml, deploy-cf.yml, Dockerfile.cloudflare, docker-compose.yml, Makefile, ecosystem.config.js, scripts/manage.sh) already targeted root app/. apps/backend/ existed only as documentation debt promised by ADR-0006 to be collapsed in a future PR. This PR collapses it. Specifically: - Salvage the production SearXNG settings.yml from apps/backend/searxng/ to searxng/settings.yml (fixes the SearXNG default-config bug). - git rm -r apps/backend/ (130+ files of stale code). - Delete .github/workflows/deploy.yml — Railway deploy is sunset by ADR-0010 in favor of CF Containers GA. The active deploy is deploy-cf.yml which targets root. - Delete scripts/verify-setup.sh — describes a Turborepo monorepo this repo never was (packages/shared, vercel.json, src/ paths) and would fail 80% of its checks today regardless. - Delete setup_webhook_quick.sh — orphaned dev artifact, no references. - Delete unused Next.js scaffold SVGs in apps/web/public/. Housekeeping: - Add pnpm-workspace.yaml declaring apps/* + workers — SDK packages reference each other via workspace:* but the workspace was never declared at root. - Add apps/sdk-ts/tests/client.test.ts — first vitest test (vitest was configured but no test files existed). - Update ADR-0006 with an amendment noting the consolidation. Original "Two backend layouts" Con line resolved. - Update docs/architecture.md, docs/configuration/env-variables.md to remove the dual-backend references. apps/backend/.env was left in working tree (gitignored, contains secrets); the user should manually compare against root .env and remove if redundant. --- .github/workflows/deploy.yml | 137 - apps/backend/ADVANCED_FEATURES.md | 688 ---- .../COMPLETE_CRAWL4AI_IMPLEMENTATION.md | 436 --- ...COMPREHENSIVE_CRAWL4AI_MISSING_FEATURES.md | 413 --- apps/backend/CRAWL4AI_INTEGRATION.md | 341 -- apps/backend/Dockerfile | 53 - apps/backend/FINAL_CRAWL4AI_AUDIT_COMPLETE.md | 406 --- apps/backend/Procfile | 5 - apps/backend/README_BACKEND.md | 66 - apps/backend/alembic.ini | 99 - apps/backend/alembic/env.py | 90 - apps/backend/alembic/script.py.mako | 24 - .../alembic/versions/001_initial_schema.py | 176 - .../versions/002_add_user_billing_tables.py | 261 -- apps/backend/app/__init__.py | 0 apps/backend/app/api/__init__.py | 0 apps/backend/app/api/dependencies.py | 118 - apps/backend/app/api/v1/__init__.py | 3 - apps/backend/app/api/v1/auth.py | 367 --- apps/backend/app/api/v1/billing.py | 413 --- apps/backend/app/api/v1/enhanced_search.py | 783 ----- apps/backend/app/api/v1/search.py | 495 --- apps/backend/app/api/v2/__init__.py | 1 - apps/backend/app/api/v2/advanced_endpoints.py | 1347 -------- apps/backend/app/config.py | 200 -- apps/backend/app/main.py | 323 -- apps/backend/app/middleware/rate_limit.py | 276 -- apps/backend/app/models/__init__.py | 39 - apps/backend/app/models/auth_models.py | 188 -- apps/backend/app/models/database.py | 188 -- apps/backend/app/models/requests.py | 479 --- apps/backend/app/models/responses.py | 214 -- apps/backend/app/models/users.py | 351 -- apps/backend/app/services/__init__.py | 384 --- apps/backend/app/services/actions_system.py | 575 ---- .../backend/app/services/adaptive_crawling.py | 611 ---- apps/backend/app/services/ai_extraction.py | 747 ----- .../app/services/attributes_extraction.py | 700 ---- apps/backend/app/services/auth_service.py | 433 --- apps/backend/app/services/batch_operations.py | 628 ---- apps/backend/app/services/browser_config.py | 578 ---- apps/backend/app/services/browser_profiler.py | 513 --- apps/backend/app/services/cache.py | 433 --- apps/backend/app/services/cache_context.py | 532 --- apps/backend/app/services/change_tracking.py | 709 ---- .../app/services/chunking_strategies.py | 593 ---- apps/backend/app/services/content_filters.py | 647 ---- apps/backend/app/services/crawl_management.py | 829 ----- apps/backend/app/services/crawler_monitor.py | 568 ---- apps/backend/app/services/database.py | 464 --- apps/backend/app/services/database_manager.py | 653 ---- apps/backend/app/services/deep_crawling.py | 803 ----- apps/backend/app/services/dispatcher.py | 664 ---- .../backend/app/services/enhanced_scraping.py | 643 ---- .../app/services/extraction_strategies.py | 664 ---- apps/backend/app/services/html_converter.py | 631 ---- apps/backend/app/services/link_analysis.py | 686 ---- apps/backend/app/services/link_preview.py | 672 ---- .../backend/app/services/llm_configuration.py | 503 --- .../app/services/markdown_generation.py | 665 ---- .../app/services/multi_engine_scraper.py | 719 ----- .../app/services/multi_entity_extraction.py | 828 ----- apps/backend/app/services/multi_search.py | 390 --- apps/backend/app/services/pdf_processing.py | 617 ---- apps/backend/app/services/proxy_rotation.py | 534 --- apps/backend/app/services/puppeteer_client.py | 33 - apps/backend/app/services/scraping.py | 815 ----- apps/backend/app/services/searxng.py | 395 --- apps/backend/app/services/stripe_service.py | 587 ---- apps/backend/app/services/table_extraction.py | 751 ----- apps/backend/app/services/url_seeder.py | 619 ---- .../app/services/user_agent_generator.py | 639 ---- .../backend/app/services/virtual_scrolling.py | 627 ---- .../app/services/webhook_integration.py | 662 ---- apps/backend/app/services/website_mapping.py | 772 ----- apps/backend/app/services/zero_retention.py | 599 ---- apps/backend/app/utils/__init__.py | 92 - apps/backend/app/utils/error_handlers.py | 302 -- apps/backend/app/utils/exceptions.py | 190 -- apps/backend/app/utils/security.py | 308 -- apps/backend/app/utils/text_processing.py | 321 -- apps/backend/app/utils/validators.py | 367 --- apps/backend/app/workers/__init__.py | 0 apps/backend/app/workers/tasks.py | 236 -- apps/backend/docker-compose.prod.yml | 88 - apps/backend/docker-compose.yml | 174 - apps/backend/env.example | 57 - .../monitoring/docker-compose.monitoring.yml | 134 - .../dashboards/searchscrape-overview.json | 627 ---- .../provisioning/dashboards/dashboards.yml | 13 - .../provisioning/datasources/prometheus.yml | 32 - .../monitoring/prometheus/prometheus.yml | 97 - apps/backend/nginx/nginx.conf | 145 - apps/backend/package.json | 16 - apps/backend/poetry.lock | 127 - apps/backend/pyproject.toml | 18 - apps/backend/pytest.ini | 34 - apps/backend/railway.json | 48 - apps/backend/requirements.txt | 303 -- apps/backend/scripts/backup.sh | 379 --- apps/backend/scripts/deploy.sh | 107 - apps/backend/scripts/health_check.sh | 97 - apps/backend/scripts/monitor.sh | 130 - apps/backend/scripts/restore.sh | 521 --- apps/backend/scripts/setup-stripe.sh | 265 -- apps/backend/scripts/setup.sh | 93 - apps/backend/scripts/start-all.sh | 79 - apps/backend/scripts/test.sh | 72 - apps/backend/searxng/settings.yml | 2841 ---------------- apps/backend/test_advanced_integration.py | 324 -- apps/backend/tests/conftest.py | 178 - apps/backend/tests/e2e/test_complete_flows.py | 620 ---- apps/backend/tests/integration/test_api.py | 244 -- .../tests/integration/test_endpoints.py | 398 --- apps/backend/tests/performance/locustfile.py | 426 --- .../tests/performance/test_benchmarks.py | 504 --- apps/backend/tests/performance/test_load.py | 400 --- apps/backend/tests/smoke/test_smoke.py | 288 -- apps/backend/tests/unit/test_models.py | 204 -- apps/backend/tests/unit/test_services.py | 361 --- apps/backend/tests/unit/test_utils.py | 315 -- apps/sdk-ts/tests/client.test.ts | 46 + apps/web/public/file.svg | 1 - apps/web/public/globe.svg | 1 - apps/web/public/next.svg | 1 - apps/web/public/vercel.svg | 1 - apps/web/public/window.svg | 1 - .../0006-monorepo-with-apps-and-workers.md | 27 +- docs/architecture.md | 5 +- docs/configuration/env-variables.md | 2 +- pnpm-workspace.yaml | 3 + scripts/verify-setup.sh | 238 -- searxng/settings.yml | 2851 ++++++++++++++++- setup_webhook_quick.sh | 55 - 134 files changed, 2898 insertions(+), 48004 deletions(-) delete mode 100644 .github/workflows/deploy.yml delete mode 100644 apps/backend/ADVANCED_FEATURES.md delete mode 100644 apps/backend/COMPLETE_CRAWL4AI_IMPLEMENTATION.md delete mode 100644 apps/backend/COMPREHENSIVE_CRAWL4AI_MISSING_FEATURES.md delete mode 100644 apps/backend/CRAWL4AI_INTEGRATION.md delete mode 100644 apps/backend/Dockerfile delete mode 100644 apps/backend/FINAL_CRAWL4AI_AUDIT_COMPLETE.md delete mode 100644 apps/backend/Procfile delete mode 100644 apps/backend/README_BACKEND.md delete mode 100644 apps/backend/alembic.ini delete mode 100644 apps/backend/alembic/env.py delete mode 100644 apps/backend/alembic/script.py.mako delete mode 100644 apps/backend/alembic/versions/001_initial_schema.py delete mode 100644 apps/backend/alembic/versions/002_add_user_billing_tables.py delete mode 100644 apps/backend/app/__init__.py delete mode 100644 apps/backend/app/api/__init__.py delete mode 100644 apps/backend/app/api/dependencies.py delete mode 100644 apps/backend/app/api/v1/__init__.py delete mode 100644 apps/backend/app/api/v1/auth.py delete mode 100644 apps/backend/app/api/v1/billing.py delete mode 100644 apps/backend/app/api/v1/enhanced_search.py delete mode 100644 apps/backend/app/api/v1/search.py delete mode 100644 apps/backend/app/api/v2/__init__.py delete mode 100644 apps/backend/app/api/v2/advanced_endpoints.py delete mode 100644 apps/backend/app/config.py delete mode 100644 apps/backend/app/main.py delete mode 100644 apps/backend/app/middleware/rate_limit.py delete mode 100644 apps/backend/app/models/__init__.py delete mode 100644 apps/backend/app/models/auth_models.py delete mode 100644 apps/backend/app/models/database.py delete mode 100644 apps/backend/app/models/requests.py delete mode 100644 apps/backend/app/models/responses.py delete mode 100644 apps/backend/app/models/users.py delete mode 100644 apps/backend/app/services/__init__.py delete mode 100644 apps/backend/app/services/actions_system.py delete mode 100644 apps/backend/app/services/adaptive_crawling.py delete mode 100644 apps/backend/app/services/ai_extraction.py delete mode 100644 apps/backend/app/services/attributes_extraction.py delete mode 100644 apps/backend/app/services/auth_service.py delete mode 100644 apps/backend/app/services/batch_operations.py delete mode 100644 apps/backend/app/services/browser_config.py delete mode 100644 apps/backend/app/services/browser_profiler.py delete mode 100644 apps/backend/app/services/cache.py delete mode 100644 apps/backend/app/services/cache_context.py delete mode 100644 apps/backend/app/services/change_tracking.py delete mode 100644 apps/backend/app/services/chunking_strategies.py delete mode 100644 apps/backend/app/services/content_filters.py delete mode 100644 apps/backend/app/services/crawl_management.py delete mode 100644 apps/backend/app/services/crawler_monitor.py delete mode 100644 apps/backend/app/services/database.py delete mode 100644 apps/backend/app/services/database_manager.py delete mode 100644 apps/backend/app/services/deep_crawling.py delete mode 100644 apps/backend/app/services/dispatcher.py delete mode 100644 apps/backend/app/services/enhanced_scraping.py delete mode 100644 apps/backend/app/services/extraction_strategies.py delete mode 100644 apps/backend/app/services/html_converter.py delete mode 100644 apps/backend/app/services/link_analysis.py delete mode 100644 apps/backend/app/services/link_preview.py delete mode 100644 apps/backend/app/services/llm_configuration.py delete mode 100644 apps/backend/app/services/markdown_generation.py delete mode 100644 apps/backend/app/services/multi_engine_scraper.py delete mode 100644 apps/backend/app/services/multi_entity_extraction.py delete mode 100644 apps/backend/app/services/multi_search.py delete mode 100644 apps/backend/app/services/pdf_processing.py delete mode 100644 apps/backend/app/services/proxy_rotation.py delete mode 100644 apps/backend/app/services/puppeteer_client.py delete mode 100644 apps/backend/app/services/scraping.py delete mode 100644 apps/backend/app/services/searxng.py delete mode 100644 apps/backend/app/services/stripe_service.py delete mode 100644 apps/backend/app/services/table_extraction.py delete mode 100644 apps/backend/app/services/url_seeder.py delete mode 100644 apps/backend/app/services/user_agent_generator.py delete mode 100644 apps/backend/app/services/virtual_scrolling.py delete mode 100644 apps/backend/app/services/webhook_integration.py delete mode 100644 apps/backend/app/services/website_mapping.py delete mode 100644 apps/backend/app/services/zero_retention.py delete mode 100644 apps/backend/app/utils/__init__.py delete mode 100644 apps/backend/app/utils/error_handlers.py delete mode 100644 apps/backend/app/utils/exceptions.py delete mode 100644 apps/backend/app/utils/security.py delete mode 100644 apps/backend/app/utils/text_processing.py delete mode 100644 apps/backend/app/utils/validators.py delete mode 100644 apps/backend/app/workers/__init__.py delete mode 100644 apps/backend/app/workers/tasks.py delete mode 100644 apps/backend/docker-compose.prod.yml delete mode 100644 apps/backend/docker-compose.yml delete mode 100644 apps/backend/env.example delete mode 100644 apps/backend/monitoring/docker-compose.monitoring.yml delete mode 100644 apps/backend/monitoring/grafana/dashboards/searchscrape-overview.json delete mode 100644 apps/backend/monitoring/grafana/provisioning/dashboards/dashboards.yml delete mode 100644 apps/backend/monitoring/grafana/provisioning/datasources/prometheus.yml delete mode 100644 apps/backend/monitoring/prometheus/prometheus.yml delete mode 100644 apps/backend/nginx/nginx.conf delete mode 100644 apps/backend/package.json delete mode 100644 apps/backend/poetry.lock delete mode 100644 apps/backend/pyproject.toml delete mode 100644 apps/backend/pytest.ini delete mode 100644 apps/backend/railway.json delete mode 100644 apps/backend/requirements.txt delete mode 100755 apps/backend/scripts/backup.sh delete mode 100755 apps/backend/scripts/deploy.sh delete mode 100755 apps/backend/scripts/health_check.sh delete mode 100755 apps/backend/scripts/monitor.sh delete mode 100755 apps/backend/scripts/restore.sh delete mode 100755 apps/backend/scripts/setup-stripe.sh delete mode 100755 apps/backend/scripts/setup.sh delete mode 100644 apps/backend/scripts/start-all.sh delete mode 100755 apps/backend/scripts/test.sh delete mode 100644 apps/backend/searxng/settings.yml delete mode 100644 apps/backend/test_advanced_integration.py delete mode 100644 apps/backend/tests/conftest.py delete mode 100644 apps/backend/tests/e2e/test_complete_flows.py delete mode 100644 apps/backend/tests/integration/test_api.py delete mode 100644 apps/backend/tests/integration/test_endpoints.py delete mode 100644 apps/backend/tests/performance/locustfile.py delete mode 100644 apps/backend/tests/performance/test_benchmarks.py delete mode 100644 apps/backend/tests/performance/test_load.py delete mode 100644 apps/backend/tests/smoke/test_smoke.py delete mode 100644 apps/backend/tests/unit/test_models.py delete mode 100644 apps/backend/tests/unit/test_services.py delete mode 100644 apps/backend/tests/unit/test_utils.py create mode 100644 apps/sdk-ts/tests/client.test.ts delete mode 100644 apps/web/public/file.svg delete mode 100644 apps/web/public/globe.svg delete mode 100644 apps/web/public/next.svg delete mode 100644 apps/web/public/vercel.svg delete mode 100644 apps/web/public/window.svg create mode 100644 pnpm-workspace.yaml delete mode 100755 scripts/verify-setup.sh delete mode 100755 setup_webhook_quick.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 2c8b814..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: Deploy to Production - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - # Test jobs - test-backend: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./apps/backend - - services: - postgres: - image: postgres:14 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: unsearch_test - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - redis: - image: redis:7 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Run tests - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/unsearch_test - REDIS_URL: redis://localhost:6379 - SECRET_KEY: test-secret-key - ENVIRONMENT: test - run: | - python -m pytest tests/ -v - - test-frontend: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./apps/web - - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - cache: 'npm' - - - name: Install dependencies - run: npm ci - - - name: Run type checking - run: npm run type-check - - - name: Run linting - run: npm run lint - - - name: Build application - run: npm run build - env: - NEXT_PUBLIC_API_URL: https://unsearch-api.railway.app - - # Deploy backend to Railway - deploy-backend: - needs: [test-backend] - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - - steps: - - uses: actions/checkout@v4 - - - name: Install Railway CLI - run: npm install -g @railway/cli - - - name: Deploy to Railway - env: - RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }} - run: | - cd apps/backend - railway login --token $RAILWAY_TOKEN - railway deploy --service backend - - # Deploy frontend to Vercel - deploy-frontend: - needs: [test-frontend] - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - cache: 'npm' - - - name: Install Vercel CLI - run: npm install -g vercel - - - name: Deploy to Vercel - env: - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} - run: | - cd apps/web - vercel --prod --token $VERCEL_TOKEN --yes diff --git a/apps/backend/ADVANCED_FEATURES.md b/apps/backend/ADVANCED_FEATURES.md deleted file mode 100644 index 32fc5a0..0000000 --- a/apps/backend/ADVANCED_FEATURES.md +++ /dev/null @@ -1,688 +0,0 @@ -# 🔥 Advanced Firecrawl-Inspired Features - -This document describes the advanced search and scraping capabilities that have been integrated into the UnSearch backend, inspired by Firecrawl's cutting-edge architecture. - -## 📋 Overview - -The backend now includes **13 major advanced features** that significantly enhance its search and scraping capabilities: - -1. **Multi-Provider Search Integration** - Intelligent search with automatic fallback -2. **Multi-Engine Scraping Architecture** - Advanced engine selection and fallback -3. **LLM-Powered Configuration** - Natural language to configuration conversion -4. **Advanced Batch Processing** - Sophisticated job management and processing -5. **Multi-Entity Extraction** - Cross-URL entity discovery and relationship mapping -6. **Browser Actions System** - Complete browser automation and interaction -7. **Website Mapping** - Fast and comprehensive URL discovery -8. **Change Tracking** - Advanced content monitoring and diff analysis -9. **Attributes Extraction** - Sophisticated HTML attribute extraction -10. **Enhanced API Endpoints** - Comprehensive v1 and v2 API enhancements -11. **Intelligent Dispatching** - Memory-adaptive resource management -12. **Advanced Content Processing** - Enhanced markdown, chunking, and analysis -13. **Comprehensive Analytics** - Deep performance monitoring and statistics - ---- - -## 🔍 1. Multi-Provider Search Integration - -**Location**: `app/services/multi_search.py` - -### Features - -- **Multiple Search Providers**: Fire Engine, Serper, SearchAPI, SearXNG, Google -- **Intelligent Fallback**: Automatic provider switching on failures -- **Performance Monitoring**: Real-time provider statistics and health checks -- **Rate Limiting**: Per-provider rate limiting with smart throttling - -### API Endpoint - -``` -POST /api/v1/v2/advanced/search/multi-provider -``` - -### Configuration - -```env -FIRE_ENGINE_BETA_URL=https://your-fire-engine-url -SERPER_API_KEY=your_serper_key -SEARCHAPI_API_KEY=your_searchapi_key -SEARXNG_ENDPOINT=http://localhost:8080 -``` - -### Example Usage - -```python -from app.services.multi_search import get_multi_search_service, SearchOptions - -service = await get_multi_search_service() -options = SearchOptions( - query="artificial intelligence", - num_results=10, - lang="en", - country="us" -) -results = await service.search(options) -``` - ---- - -## 🤖 2. Multi-Engine Scraping Architecture - -**Location**: `app/services/multi_engine_scraper.py` - -### Features - -- **Multiple Scraping Engines**: Index, Fire Engine variants, Playwright, Fetch, PDF, DOCX -- **Intelligent Engine Selection**: Based on content type, capabilities, and performance -- **Advanced Capabilities**: Actions, screenshots, mobile simulation, stealth mode -- **Automatic Fallback**: Smart fallback chain for maximum reliability - -### Available Engines - -- `index` - Pre-cached content (highest priority) -- `fire-engine;chrome-cdp` - Chrome DevTools Protocol -- `fire-engine;chrome-cdp;stealth` - Stealth mode Chrome CDP -- `fire-engine;playwright` - Playwright integration -- `fire-engine;tlsclient` - TLS client for advanced scenarios -- `playwright` - Direct Playwright service -- `fetch` - Basic HTTP fetch (fallback) -- `pdf` - PDF document processing -- `docx` - Word document processing - -### API Endpoint - -``` -POST /api/v1/v2/advanced/scrape/multi-engine -``` - -### Example Usage - -```python -from app.services.multi_engine_scraper import get_multi_engine_service - -service = await get_multi_engine_service() -result = await service.scrape( - url="https://example.com", - config=scraping_config, - preferred_engine=EngineType.FIRE_ENGINE_CDP, - required_capabilities=["screenshot", "actions"] -) -``` - ---- - -## 🧠 3. LLM-Powered Configuration - -**Location**: `app/services/llm_configuration.py` - -### Features - -- **Natural Language Processing**: Convert descriptions to structured configurations -- **Multiple Configuration Types**: Crawler options, extraction schemas, content filters, search strategies -- **Validation**: Automatic validation of generated configurations -- **Multiple Models**: Support for GPT-4, GPT-4-turbo, GPT-3.5-turbo with fallbacks - -### Configuration Types - -- **Crawler Options**: URL patterns, depth limits, crawling behavior -- **Extraction Schemas**: JSON schemas for structured data extraction -- **Content Filters**: Relevance filtering and content selection rules -- **Search Strategies**: Search engine selection and optimization - -### API Endpoint - -``` -POST /api/v1/v2/advanced/config/generate -``` - -### Example Usage - -```python -from app.services.llm_configuration import generate_config_from_prompt - -config = await generate_config_from_prompt( - prompt="Crawl a blog site and extract only the article pages, excluding navigation", - config_type="crawler" -) -``` - -### Example Prompts - -- _"Crawl an e-commerce site and extract product information including prices and reviews"_ -- _"Filter content to only include technical articles about machine learning"_ -- _"Search for recent news articles from reliable sources in the past week"_ - ---- - -## 📦 4. Advanced Batch Processing - -**Location**: `app/services/batch_operations.py` - -### Features - -- **Multiple Operation Types**: Batch scraping, search, extraction, crawling -- **Intelligent Job Scheduling**: Priority-based queue with resource management -- **Progress Tracking**: Real-time progress updates and estimated completion -- **Error Handling**: Sophisticated retry logic with exponential backoff -- **Webhook Integration**: Status update notifications -- **Job Control**: Pause, resume, cancel operations - -### API Endpoints - -``` -POST /api/v1/v2/advanced/batch/submit -GET /api/v1/v2/advanced/batch/{job_id}/status -POST /api/v1/v2/advanced/batch/{job_id}/control -``` - -### Example Usage - -```python -from app.services.batch_operations import get_batch_service - -service = await get_batch_service() -job_id = await service.submit_batch_scrape( - urls=["https://example1.com", "https://example2.com"], - priority=10, - webhook_url="https://your-app.com/webhook" -) -``` - ---- - -## 🔗 5. Multi-Entity Extraction - -**Location**: `app/services/multi_entity_extraction.py` - -### Features - -- **Cross-URL Entity Discovery**: Find related URLs and extract linked entities -- **Multiple Extraction Strategies**: Linked entities, hierarchical, semantic similarity, temporal, cross-reference -- **Relationship Mapping**: Map relationships between entities across URLs -- **Entity Validation**: Cross-reference validation and consistency checking -- **Multiple Extraction Methods**: LLM, regex, CSS selectors with confidence scoring - -### Extraction Strategies - -- `linked_entities` - Extract entities and find related URLs -- `hierarchical` - Follow hierarchical relationships -- `semantic_similarity` - Group by semantic similarity -- `temporal_sequence` - Time-based entity relationships -- `cross_reference` - Cross-reference validation - -### API Endpoint - -``` -POST /api/v1/v2/advanced/extract/multi-entity -``` - -### Example Usage - -```python -from app.services.multi_entity_extraction import get_multi_entity_service - -service = await get_multi_entity_service() -request = MultiEntityExtractionRequest( - urls=["https://company.com/about", "https://company.com/team"], - schema={ - "type": "object", - "properties": { - "name": {"type": "string"}, - "position": {"type": "string"}, - "email": {"type": "string"} - } - }, - extraction_strategy=ExtractionStrategy.LINKED_ENTITIES -) -result = await service.extract_multi_entity(request) -``` - ---- - -## 🎯 6. Browser Actions System - -**Location**: `app/services/actions_system.py` - -### Features - -- **Complete Action Support**: All Firecrawl action types (wait, click, scroll, write, press, screenshot, scrape, executeJavascript, pdf) -- **Intelligent Browser Management**: Automatic browser initialization and cleanup -- **Multiple Browser Engines**: Playwright, Fire Engine, Puppeteer support -- **Advanced Action Sequencing**: Complex interaction workflows -- **Screenshot & PDF Generation**: High-quality captures and documents -- **JavaScript Execution**: Custom script execution within pages - -### API Endpoint - -``` -POST /api/v1/v2/advanced/actions/execute -``` - -### Action Types - -- `wait` - Wait for time or element appearance -- `click` - Click elements (single or all matching) -- `scroll` - Scroll page or specific elements -- `write` - Type text into input fields -- `press` - Press keyboard keys -- `screenshot` - Capture page screenshots -- `scrape` - Extract current page content -- `executeJavascript` - Run custom JavaScript -- `pdf` - Generate PDF of current page - -### Example Usage - -```python -from app.services.actions_system import execute_browser_actions - -actions = [ - {"type": "wait", "milliseconds": 3000}, - {"type": "click", "selector": "#search-button"}, - {"type": "write", "text": "AI trends 2024"}, - {"type": "press", "key": "Enter"}, - {"type": "wait", "selector": ".search-results"}, - {"type": "screenshot", "fullPage": True}, - {"type": "scrape"} -] - -result = await execute_browser_actions("https://example.com", actions) -print(f"Executed {len(actions)} actions - Success: {result.success}") -``` - ---- - -## 🗺️ 7. Website Mapping - -**Location**: `app/services/website_mapping.py` - -### Features - -- **Multi-Strategy Discovery**: Sitemaps, search engines, crawling, index lookup -- **Fast URL Enumeration**: Discover thousands of URLs quickly -- **Advanced Filtering**: Subdomain, path, pattern-based filtering -- **Search Integration**: Use search engines for comprehensive discovery -- **Sitemap Intelligence**: Parse XML sitemaps and robots.txt -- **Metadata Extraction**: Page titles, descriptions, priorities, last modified - -### API Endpoint - -``` -POST /api/v1/v2/advanced/map/website -``` - -### Mapping Strategies - -- `sitemap_only` - Use XML sitemaps exclusively -- `search_engine` - Use search engine site: queries -- `combined` - Use both sitemaps and search engines (default) -- `crawl_based` - Use web crawling discovery - -### Example Usage - -```python -from app.services.website_mapping import map_website_urls - -result = await map_website_urls( - url="https://example.com", - strategy="combined", - limit=1000, - include_subdomains=True -) - -print(f"Discovered {result.total_urls} URLs from {len(result.sources_breakdown)} sources") -for source, count in result.sources_breakdown.items(): - print(f" {source}: {count} URLs") -``` - ---- - -## 📊 8. Change Tracking - -**Location**: `app/services/change_tracking.py` - -### Features - -- **Content Comparison**: Advanced diff generation between versions -- **Change Detection**: Percentage-based change calculation -- **Historical Tracking**: Store and compare multiple versions -- **Smart Notifications**: Webhook alerts for significant changes -- **Multi-Format Diffs**: Text, HTML, JSON diff formats -- **Change Analytics**: Identify significant vs. minor changes - -### API Endpoint - -``` -POST /api/v1/v2/advanced/track/changes -``` - -### Change Status Types - -- `new` - First time tracking this URL -- `same` - No changes detected -- `changed` - Content has changed -- `removed` - Content no longer accessible - -### Example Usage - -```python -from app.services.change_tracking import track_url_changes - -result = await track_url_changes( - url="https://example.com/news", - tag="news-monitoring", - threshold=0.05, - webhook_url="https://yourapp.com/webhook" -) - -print(f"Change status: {result.tracking_data.change_status.value}") -if result.tracking_data.change_percentage > 0: - print(f"Change percentage: {result.tracking_data.change_percentage:.1f}%") -``` - ---- - -## 🔍 9. Attributes Extraction - -**Location**: `app/services/attributes_extraction.py` - -### Features - -- **CSS Selector-Based**: Extract any HTML attribute using CSS selectors -- **Multi-Processing Types**: Raw, cleaned, URLs resolved, numeric, boolean, list -- **Advanced Filtering**: Empty value filtering, duplicate removal, validation -- **Bulk Extraction**: Process multiple selectors and attributes -- **Context Awareness**: Include element context and metadata -- **URL Resolution**: Automatically resolve relative URLs - -### API Endpoint - -``` -POST /api/v1/v2/advanced/extract/attributes -``` - -### Processing Types - -- `raw` - Extract values as-is -- `cleaned` - Clean and normalize text -- `urls_resolved` - Resolve relative URLs to absolute -- `numeric` - Extract numeric values -- `boolean` - Convert to boolean values -- `list` - Split into lists using delimiters - -### Example Usage - -```python -from app.services.attributes_extraction import extract_page_attributes - -result = await extract_page_attributes( - url="https://example.com", - selector_attribute_pairs=[ - ("a", "href"), # Extract all links - ("img", "src"), # Extract all image sources - ("meta[name]", "content") # Extract meta tag content - ], - processing_type="urls_resolved" -) - -print(f"Extracted {result.total_attributes_extracted} attributes from {result.total_elements_processed} elements") -``` - ---- - -## 🌐 10. Enhanced API Endpoints - -### V1 Enhanced Endpoints - -**Location**: `app/api/v1/enhanced_search.py` - -- Enhanced search with all advanced features integrated -- Backward-compatible with existing API -- Extended configuration options - -### V2 Advanced Endpoints - -**Location**: `app/api/v2/advanced_endpoints.py` - -- **Multi-Provider Search**: `POST /v2/advanced/search/multi-provider` -- **Multi-Engine Scraping**: `POST /v2/advanced/scrape/multi-engine` -- **LLM Configuration**: `POST /v2/advanced/config/generate` -- **Batch Operations**: `POST /v2/advanced/batch/submit` -- **Multi-Entity Extraction**: `POST /v2/advanced/extract/multi-entity` -- **Browser Actions**: `POST /v2/advanced/actions/execute` -- **Website Mapping**: `POST /v2/advanced/map/website` -- **Change Tracking**: `POST /v2/advanced/track/changes` -- **Attributes Extraction**: `POST /v2/advanced/extract/attributes` -- **Advanced Scraping**: `POST /v2/advanced/scrape/advanced` -- **Comprehensive Stats**: `GET /v2/advanced/stats/comprehensive` -- **Health Check**: `GET /v2/advanced/health/advanced` - ---- - -## ⚡ 7. Performance & Monitoring - -### Comprehensive Statistics - -All services provide detailed performance metrics: - -```python -# Get comprehensive stats for all services -GET /api/v1/v2/advanced/stats/comprehensive -``` - -### Service Health Monitoring - -```python -# Check health of all advanced services -GET /api/v1/v2/advanced/health/advanced -``` - -### Individual Service Stats - -- Multi-search provider performance and availability -- Multi-engine success rates and processing times -- LLM configuration usage and token consumption -- Batch operation queue status and throughput -- Entity extraction accuracy and cache hit rates - ---- - -## 🔧 Configuration - -### Environment Variables - -```env -# Fire Engine (Advanced Scraping) -FIRE_ENGINE_BETA_URL=https://your-fire-engine-instance -FIRE_ENGINE_TIMEOUT=60 - -# Multi-Provider Search APIs -SERPER_API_KEY=your_serper_api_key -SEARCHAPI_API_KEY=your_searchapi_key - -# LLM Configuration -OPENAI_API_KEY=your_openai_api_key -OPENAI_MODEL=gpt-4 -OPENAI_MAX_TOKENS=4096 - -# Batch Operations -BATCH_MAX_CONCURRENT_JOBS=5 -BATCH_MAX_WORKERS=10 -BATCH_JOB_TIMEOUT=3600 - -# Playwright Service -PLAYWRIGHT_SERVICE_URL=http://localhost:3000 -PLAYWRIGHT_TIMEOUT=60 -``` - ---- - -## 🚀 Getting Started - -### 1. Install Dependencies - -The new features integrate seamlessly with existing dependencies. No additional installations required. - -### 2. Update Configuration - -Add the environment variables for the services you want to enable: - -```bash -# Copy the example environment file -cp apps/backend/env.example apps/backend/.env - -# Edit with your API keys and service URLs -nano apps/backend/.env -``` - -### 3. Test the Integration - -```bash -# Run the comprehensive integration test -cd apps/backend -python test_advanced_integration.py -``` - -### 4. Start Using Advanced Features - -```python -# Example: Multi-provider search with result scraping -curl -X POST "http://localhost:8000/api/v1/v2/advanced/search/multi-provider" \ - -H "X-API-Key: your-api-key" \ - -H "Content-Type: application/json" \ - -d '{ - "query": "artificial intelligence trends 2024", - "num_results": 10, - "scrape_results": true, - "scrape_config": { - "extract_text": true, - "extract_images": true - } - }' -``` - ---- - -## 🎯 Advanced Use Cases - -### 1. Comprehensive Content Research - -```python -# Use multi-entity extraction for research across related pages -POST /v2/advanced/extract/multi-entity -{ - "urls": ["https://company.com/about"], - "schema": { - "type": "object", - "properties": { - "leadership": {"type": "array"}, - "products": {"type": "array"}, - "locations": {"type": "array"} - } - }, - "follow_links": true, - "max_related_urls": 50 -} -``` - -### 2. Natural Language Scraping Configuration - -```python -# Generate scraping configuration from natural language -POST /v2/advanced/config/generate -{ - "prompt": "I want to scrape a news website. Extract article titles, authors, publication dates, and full content. Ignore ads, navigation, and comments.", - "config_type": "crawler" -} -``` - -### 3. Large-Scale Data Collection - -```python -# Submit batch job for processing hundreds of URLs -POST /v2/advanced/batch/submit -{ - "operation_type": "scrape", - "urls": ["https://site1.com", "https://site2.com", ...], - "config": { - "extract_text": true, - "javascript_rendering": true, - "stealth_mode": true - }, - "webhook_url": "https://yourapp.com/batch-complete" -} -``` - ---- - -## 🛠 Troubleshooting - -### Common Issues - -1. **LLM Configuration Not Working** - - Ensure `OPENAI_API_KEY` is set - - Check OpenAI account has sufficient credits - - Verify model access permissions - -2. **Multi-Provider Search Returns No Results** - - Check that at least one search provider is configured - - Verify API keys are valid and have quota - - Check network connectivity to provider endpoints - -3. **Multi-Engine Scraping Fails** - - Verify Fire Engine or Playwright services are running - - Check service URLs are accessible - - Ensure sufficient system resources for concurrent scraping - -4. **Batch Operations Stuck** - - Check batch service is started: `await get_batch_service()` - - Verify worker tasks are running - - Check job queue for errors in logs - -### Debug Mode - -Enable debug logging to troubleshoot issues: - -```env -DEBUG=true -LOG_LEVEL=DEBUG -``` - ---- - -## 📈 Performance Optimization - -### Recommended Settings - -```env -# For high-volume production use -SCRAPING_MAX_CONCURRENT=20 -BATCH_MAX_CONCURRENT_JOBS=10 -BATCH_MAX_WORKERS=20 - -# For memory-constrained environments -SCRAPING_MAX_CONCURRENT=5 -BATCH_MAX_CONCURRENT_JOBS=3 -BATCH_MAX_WORKERS=5 -``` - -### Monitoring - -Monitor resource usage and adjust concurrency limits based on: - -- Available system memory -- Network bandwidth -- Provider rate limits -- Database connection pool size - ---- - -## 🎉 Success! - -Your UnSearch backend now includes all the advanced features found in Firecrawl and more! The implementation provides: - -- **9 Major Advanced Features** with comprehensive functionality -- **Seamless Integration** with existing codebase -- **Production-Ready** architecture with proper error handling -- **Extensive Documentation** and examples -- **Full Test Coverage** with integration tests - -You now have one of the most advanced web search and scraping backends available, combining the best of Firecrawl's capabilities with your existing UnSearch architecture! 🚀 diff --git a/apps/backend/COMPLETE_CRAWL4AI_IMPLEMENTATION.md b/apps/backend/COMPLETE_CRAWL4AI_IMPLEMENTATION.md deleted file mode 100644 index 746919e..0000000 --- a/apps/backend/COMPLETE_CRAWL4AI_IMPLEMENTATION.md +++ /dev/null @@ -1,436 +0,0 @@ -# 🎉 Complete Crawl4AI Implementation - Final Report - -## 📊 **Executive Summary** - -**MISSION ACCOMPLISHED**: All sophisticated crawl4ai features have been successfully implemented and integrated into the backend system. The backend now provides **complete feature parity** with crawl4ai plus additional production-ready capabilities. - -**Total Implementation**: **15 major components** with **80+ advanced features** across **12 new service modules** and **4 enhanced API endpoints**. - ---- - -## 🚀 **Complete Feature Implementation Matrix** - -### ✅ **Core Extraction & Processing** - -| Feature | Status | Implementation | -| ------------------------- | ----------- | -------------------------------------------------------------- | -| **Extraction Strategies** | ✅ Complete | 5 strategies: Cosine, JsonCSS, Regex, LLM, NoExtraction | -| **Content Filtering** | ✅ Complete | 4 filters: BM25, Pruning, LLM, NoFilter | -| **Markdown Generation** | ✅ Complete | Citations, link analysis, multiple formats | -| **Text Chunking** | ✅ Complete | 6 strategies: Regex, Sentence, Paragraph, Fixed, Topic, Hybrid | -| **Table Extraction** | ✅ Complete | 4 strategies: Default, LLM, Smart, None | - -### ✅ **Intelligence & Learning** - -| Feature | Status | Implementation | -| --------------------- | ----------- | ------------------------------------------------------------- | -| **Adaptive Crawling** | ✅ Complete | Statistical learning, state persistence, saturation detection | -| **Link Analysis** | ✅ Complete | 3-layer scoring: relevance, authority, quality, freshness | -| **Virtual Scrolling** | ✅ Complete | Infinite scroll detection, smart waiting, content extraction | -| **URL Seeding** | ✅ Complete | Sitemap parsing, crawl discovery, pattern filtering | - -### ✅ **Infrastructure & Performance** - -| Feature | Status | Implementation | -| ------------------------- | ----------- | ---------------------------------------------------------- | -| **Dispatcher System** | ✅ Complete | Memory-adaptive, semaphore-based, rate limiting | -| **Browser Configuration** | ✅ Complete | Comprehensive setup, proxy, geolocation, user agents | -| **Service Registry** | ✅ Complete | Centralized component management | -| **API Endpoints** | ✅ Complete | Enhanced search, table extraction, chunking, URL discovery | - ---- - -## 🏗️ **Architecture Overview** - -### **Service Layer Structure** - -``` -app/services/ -├── Core Services -│ ├── scraping.py # Original scraping service -│ └── enhanced_scraping.py # Orchestration layer with all features -│ -├── Advanced Extraction -│ ├── extraction_strategies.py # 5 extraction strategies -│ ├── content_filters.py # 4 content filtering strategies -│ ├── chunking_strategies.py # 6 text chunking approaches -│ └── table_extraction.py # 4 table extraction methods -│ -├── Content Processing -│ ├── markdown_generation.py # Enhanced markdown with citations -│ └── link_analysis.py # 3-layer intelligent link scoring -│ -├── Crawling Intelligence -│ ├── adaptive_crawling.py # Learning-based optimization -│ ├── virtual_scrolling.py # Infinite page handling -│ └── url_seeder.py # Multi-source URL discovery -│ -├── Infrastructure -│ ├── dispatcher.py # Memory-aware concurrency -│ ├── browser_config.py # Comprehensive browser management -│ └── __init__.py # Service registry with 60+ exports -│ -└── Legacy Services (Enhanced) - ├── auth_service.py # Authentication - ├── cache.py # Caching - ├── database.py # Database operations - └── searxng.py # Search engine integration -``` - -### **API Endpoint Structure** - -``` -/enhanced/ -├── POST /search # Enhanced search with all features -├── POST /scrape # Direct scraping with advanced capabilities -├── POST /extract-tables # Table extraction endpoint -├── POST /chunk-content # Text chunking endpoint -├── POST /discover-urls # URL discovery endpoint -├── GET /features # Feature documentation -└── GET /performance # System performance metrics -``` - ---- - -## 🎯 **Feature Comparison: Crawl4AI vs Our Backend** - -| Category | Crawl4AI | Our Backend | Advantage | -| ------------------------- | ------------------ | ------------------------------ | -------------- | -| **Extraction Strategies** | ✅ 5 strategies | ✅ 5 strategies | **Equal** | -| **Content Filtering** | ✅ 3 filters | ✅ 4 filters | **Backend +1** | -| **Markdown Generation** | ✅ Basic | ✅ Enhanced with citations | **Backend** | -| **Adaptive Crawling** | ✅ Statistical | ✅ Statistical + Extensions | **Backend** | -| **Browser Management** | ✅ Playwright only | ✅ Multi-browser + configs | **Backend** | -| **Concurrency Control** | ✅ Basic | ✅ Memory-adaptive | **Backend** | -| **Production Features** | ❌ Limited | ✅ Full (auth, cache, logging) | **Backend** | -| **API Integration** | ❌ None | ✅ RESTful with documentation | **Backend** | -| **Scalability** | ❌ Single instance | ✅ Distributed ready | **Backend** | -| **Monitoring** | ❌ Basic | ✅ Comprehensive metrics | **Backend** | - -**Result**: **Backend significantly exceeds** crawl4ai capabilities while maintaining full compatibility. - ---- - -## 💡 **Advanced Implementation Highlights** - -### **1. Intelligent Extraction Pipeline** - -```python -# Multi-strategy extraction with fallbacks -extraction_strategy = "cosine" # Semantic clustering -content_filter = "bm25" # Relevance filtering -markdown_generation = True # With citations -chunking_strategy = "hybrid" # Adaptive chunking -``` - -### **2. Memory-Adaptive Dispatcher** - -```python -# Automatically adjusts concurrency based on system resources -dispatcher = MemoryAdaptiveDispatcher( - memory_threshold=80.0, # Adapt at 80% memory usage - rate_limiter=RateLimiter( # Per-domain rate limiting - max_requests=100, - time_window=60.0, - per_domain=True - ) -) -``` - -### **3. Sophisticated Table Extraction** - -```python -# Schema-based table extraction -schema = { - "baseSelector": ".product-table", - "fields": [ - {"name": "product", "selector": "td:nth-child(1)", "type": "text"}, - {"name": "price", "selector": ".price", "type": "text"}, - {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"} - ] -} -``` - -### **4. Adaptive Crawling with Learning** - -```python -# Learns crawling patterns and stops when saturation is reached -crawler = AdaptiveCrawler( - confidence_threshold=0.8, # Stop when 80% confident - strategy="statistical", # Learn from term frequencies - save_state=True # Persist learning between runs -) -``` - ---- - -## 📊 **Performance Benchmarks** - -### **Crawl4AI vs Enhanced Backend** - -| Metric | Crawl4AI | Enhanced Backend | Improvement | -| ----------------------- | -------- | ---------------- | ----------------------- | -| **Content Quality** | Baseline | +40% | BM25 filtering | -| **Extraction Accuracy** | Baseline | +65% | Multi-strategy approach | -| **Link Relevance** | Baseline | +80% | 3-layer scoring | -| **Processing Speed** | Baseline | ~Same | Intelligent caching | -| **Memory Efficiency** | Baseline | +25% | Adaptive dispatcher | -| **Error Handling** | Basic | Advanced | Graceful degradation | -| **Scalability** | Limited | High | Production architecture | - -### **System Resource Optimization** - -- **Adaptive Concurrency**: Automatically scales from 1-50 concurrent operations based on system load -- **Memory Management**: Peak memory usage reduced by 25% through intelligent resource allocation -- **Rate Limiting**: Per-domain throttling prevents overwhelming target servers -- **Caching Strategy**: Multi-level caching reduces redundant operations by 60% - ---- - -## 🛠️ **Usage Examples** - -### **Basic Enhanced Search** - -```bash -POST /enhanced/search -{ - "query": "machine learning tutorials", - "engines": ["google", "bing"], - "scrape_content": true, - "extraction_strategy": "cosine", - "extraction_config": { - "semantic_filter": "machine learning", - "top_k": 5, - "word_count_threshold": 50 - } -} -``` - -### **Advanced Table Extraction** - -```bash -POST /enhanced/extract-tables -{ - "html_content": "...
", - "strategy": "smart", - "config": { - "table_score_threshold": 8, - "extract_links": true - } -} -``` - -### **Intelligent Content Chunking** - -```bash -POST /enhanced/chunk-content -{ - "text": "Long article content...", - "strategy": "hybrid", - "config": { - "target_size": 1000, - "max_size": 2000, - "preserve_words": true - } -} -``` - -### **URL Discovery** - -```bash -POST /enhanced/discover-urls -{ - "base_url": "https://example.com", - "source": "sitemap", - "max_urls": 100, - "pattern": ".*/(blog|article)/.*", - "query": "web scraping" -} -``` - ---- - -## 🔧 **Configuration Management** - -### **Comprehensive Configuration System** - -```python -# ScrapingConfig with all features -config = { - # Basic settings - "extract_text": True, - "extract_images": True, - "extract_links": True, - - # Advanced features - "extraction_strategy": "cosine", - "extraction_config": {...}, - "content_filter": "bm25", - "content_filter_config": {...}, - "markdown_generation": True, - "markdown_config": {...}, - "adaptive_crawling": True, - "virtual_scrolling": True, - "link_analysis": True, - - # Browser configuration - "browser_config": { - "browser_type": "chromium", - "headless": True, - "viewport_width": 1920, - "viewport_height": 1080, - "user_agent_config": { - "device_type": "desktop", - "randomize": True - } - } -} -``` - ---- - -## 📈 **Monitoring & Observability** - -### **Comprehensive Performance Metrics** - -```bash -GET /enhanced/performance -{ - "timestamp": "2024-01-15T10:30:00Z", - "performance_metrics": { - "dispatcher_stats": { - "total_requests": 1250, - "successful_requests": 1185, - "success_rate": 0.948, - "avg_response_time": 2.3, - "current_active": 8 - }, - "resource_usage": { - "memory_percent": 67.2, - "cpu_percent": 45.1, - "peak_memory_usage_bytes": 1073741824 - }, - "concurrency": { - "max_concurrent": 20, - "current_max_concurrent": 15, - "adaptation_count": 12 - } - } -} -``` - ---- - -## 🚦 **Quality Assurance** - -### **Error Handling & Resilience** - -- ✅ **Graceful Degradation**: All advanced features fail safely to basic functionality -- ✅ **Comprehensive Logging**: Structured logging with request tracing -- ✅ **Rate Limiting**: Prevents overwhelming target servers -- ✅ **Resource Management**: Memory-aware operation scaling -- ✅ **Timeout Management**: Configurable timeouts for all operations -- ✅ **Retry Logic**: Smart retry with exponential backoff - -### **Testing Coverage** - -- ✅ **Unit Tests**: All extraction strategies and filters tested -- ✅ **Integration Tests**: End-to-end API endpoint testing -- ✅ **Performance Tests**: Load testing with various configurations -- ✅ **Error Handling**: Comprehensive error scenario testing - ---- - -## 🔮 **Future Enhancement Roadmap** - -### **Phase 1: Advanced AI Integration** _(Next 2-4 weeks)_ - -1. **LLM Provider Integration**: OpenAI, Anthropic, local models -2. **Embedding-based Strategies**: Vector similarity for content matching -3. **Multi-modal Processing**: Image and video content analysis - -### **Phase 2: Scalability Enhancements** _(Next 1-2 months)_ - -1. **Distributed Processing**: Multi-node crawling coordination -2. **Kubernetes Deployment**: Container orchestration -3. **Advanced Caching**: Redis cluster integration - -### **Phase 3: Intelligence Upgrades** _(Next 2-3 months)_ - -1. **Real-time Learning**: Continuous model improvement -2. **Predictive Crawling**: AI-driven URL prioritization -3. **Content Quality Prediction**: Pre-crawl quality assessment - ---- - -## 🎉 **Final Results Summary** - -### **📊 Implementation Scorecard** - -- **Features Implemented**: **✅ 100% Complete** (15/15 major components) -- **API Coverage**: **✅ 100% Complete** (All crawl4ai features + enhancements) -- **Performance**: **✅ Exceeds** crawl4ai by 40-80% across key metrics -- **Production Readiness**: **✅ Enterprise Grade** (Auth, monitoring, scaling) -- **Documentation**: **✅ Comprehensive** (Usage examples, configuration guides) - -### **🚀 Business Impact** - -- **Development Time Saved**: 3-6 months of development work completed -- **Feature Parity**: Complete crawl4ai compatibility + production enhancements -- **Scalability**: Ready for enterprise deployment from day one -- **Maintenance**: Unified codebase reduces technical debt -- **User Experience**: Enhanced APIs with comprehensive documentation - -### **🎯 Competitive Advantages** - -1. **Superior Performance**: 40-80% improvement in key metrics -2. **Production Ready**: Authentication, monitoring, caching included -3. **Scalable Architecture**: Memory-adaptive, distributed-ready -4. **Comprehensive APIs**: RESTful endpoints with full documentation -5. **Advanced Features**: Goes beyond crawl4ai with unique capabilities - ---- - -## 🔗 **Quick Start Guide** - -### **1. Basic Enhanced Scraping** - -```python -from app.services import get_enhanced_scraping_service - -async with get_enhanced_scraping_service() as scraper: - results = await scraper.scrape_urls_enhanced( - urls=["https://example.com"], - config=ScrapingConfig( - extraction_strategy="cosine", - content_filter="bm25", - markdown_generation=True - ) - ) -``` - -### **2. Advanced API Usage** - -```bash -# Complete search with all features -curl -X POST "/enhanced/search" \ - -H "Authorization: Bearer YOUR_API_KEY" \ - -d '{ - "query": "web scraping", - "extraction_strategy": "llm", - "adaptive_crawling": true, - "virtual_scrolling": true - }' -``` - -### **3. Performance Monitoring** - -```bash -# Get comprehensive metrics -curl -X GET "/enhanced/performance" \ - -H "Authorization: Bearer YOUR_API_KEY" -``` - ---- - -**🏆 CONCLUSION: The backend now provides the most advanced web scraping and content extraction platform available, combining the best of crawl4ai with production-grade enhancements and scalability. All sophisticated crawl4ai features have been successfully implemented and enhanced beyond the original specifications.** diff --git a/apps/backend/COMPREHENSIVE_CRAWL4AI_MISSING_FEATURES.md b/apps/backend/COMPREHENSIVE_CRAWL4AI_MISSING_FEATURES.md deleted file mode 100644 index bb36932..0000000 --- a/apps/backend/COMPREHENSIVE_CRAWL4AI_MISSING_FEATURES.md +++ /dev/null @@ -1,413 +0,0 @@ -# 🎯 **COMPREHENSIVE CRAWL4AI MISSING FEATURES - FULLY IMPLEMENTED** - -## 📊 **Executive Summary** - -After an exhaustive end-to-end review of both crawl4ai and our backend implementation, **7 major missing feature categories** were identified and **FULLY IMPLEMENTED**. Our backend now provides **100% feature parity** with crawl4ai plus **significant enhancements**. - -**Total Implementation**: **6 new service modules** with **40+ advanced components** and **80+ exported functions/classes**. - ---- - -## 🔍 **Missing Features Identified & Implemented** - -### ✅ **1. Deep Crawling System** - `deep_crawling.py` - -**Problem**: Missing sophisticated multi-strategy deep crawling with advanced filtering and scoring. - -**✅ IMPLEMENTED**: - -- **3 Crawling Strategies**: BFS, DFS, Best-First with priority queues -- **5 URL Filters**: Domain, Pattern, ContentType, SEO, ContentRelevance -- **5 URL Scorers**: Keyword relevance, path depth, domain authority, freshness, composite -- **Advanced Chain System**: FilterChain for combining multiple filters -- **Progress Tracking**: Comprehensive crawling statistics and metrics -- **Factory Functions**: `create_deep_crawl_strategy()`, `deep_crawl()` - -**Key Components**: - -```python -# Multiple crawling strategies -BFSDeepCrawlStrategy # Breadth-first search -DFSDeepCrawlStrategy # Depth-first search -BestFirstCrawlStrategy # Priority-based crawling - -# Sophisticated filtering -URLFilter, DomainFilter, URLPatternFilter -ContentTypeFilter, SEOFilter, ContentRelevanceFilter - -# Intelligent scoring -KeywordRelevanceScorer, PathDepthScorer, DomainAuthorityScorer -FreshnessScorer, CompositeScorer -``` - ---- - -### ✅ **2. PDF Processing System** - `pdf_processing.py` - -**Problem**: Missing comprehensive PDF document processing and analysis capabilities. - -**✅ IMPLEMENTED**: - -- **2 Processing Strategies**: NaivePDFProcessor (with PyPDF2), MockPDFProcessor -- **Complete Metadata Extraction**: Title, author, creation date, page count, etc. -- **Multi-format Output**: Raw text, HTML, Markdown conversion -- **Image Extraction**: PDF embedded images with base64 encoding -- **Page-by-page Processing**: Individual page handling with layout preservation -- **Link Extraction**: URLs and references from PDF content -- **Error Handling**: Graceful fallback for missing dependencies - -**Key Components**: - -```python -# Processing strategies -PDFProcessorStrategy, NaivePDFProcessor, MockPDFProcessor - -# Data structures -PDFMetadata, PDFPage, PDFProcessResult, PDFImage - -# Convenience functions -process_pdf_file(), extract_pdf_text(), pdf_to_markdown() -``` - ---- - -### ✅ **3. Browser Profiler System** - `browser_profiler.py` - -**Problem**: Missing identity-based crawling with persistent browser profiles. - -**✅ IMPLEMENTED**: - -- **Profile Management**: Create, list, delete, import/export profiles -- **Interactive Setup**: Browser-based profile configuration -- **Cross-platform Support**: Works on Windows, macOS, Linux -- **Profile Validation**: Health checking and cleanup of invalid profiles -- **Statistics & Analytics**: Usage tracking and performance metrics -- **BrowserConfig Integration**: Seamless integration with existing browser system - -**Key Components**: - -```python -# Core classes -BrowserProfiler, BrowserProfile - -# Factory functions -get_browser_profiler(), create_browser_profile() -get_profile_browser_config(), list_browser_profiles() -``` - ---- - -### ✅ **4. Link Preview System** - `link_preview.py` - -**Problem**: Missing advanced link head extraction and metadata analysis. - -**✅ IMPLEMENTED**: - -- **Parallel Processing**: Concurrent link metadata extraction -- **Rich Metadata Extraction**: OpenGraph, Twitter Cards, standard meta tags -- **Content Analysis**: Preview text, keyword extraction, content quality scoring -- **Advanced Filtering**: Pattern-based inclusion/exclusion, domain filtering -- **Relevance Scoring**: BM25-style query relevance calculation -- **Performance Optimization**: Caching, timeout handling, size limits - -**Key Components**: - -```python -# Core classes -LinkPreview, LinkPreviewConfig, LinkPreviewResult, LinkMetadata - -# Processing functions -extract_link_previews(), filter_links_by_quality() -``` - ---- - -### ✅ **5. Crawler Monitor System** - `crawler_monitor.py` - -**Problem**: Missing real-time crawling status tracking and performance monitoring. - -**✅ IMPLEMENTED**: - -- **Real-time Monitoring**: Live status tracking with metrics collection -- **System Resource Tracking**: CPU, memory, disk I/O, network monitoring -- **Performance Analytics**: Response times, success rates, throughput calculations -- **Alert System**: Configurable thresholds with event callbacks -- **Terminal UI Framework**: Ready for rich terminal display (extensible) -- **Comprehensive Statistics**: Success rates, error distribution, recommendations - -**Key Components**: - -```python -# Core monitoring -CrawlerMonitor, CrawlStatus, TaskMetrics, SystemMetrics, CrawlerStats - -# Convenience functions -create_crawler_monitor(), get_global_monitor() -start_global_monitoring(), stop_global_monitoring() -``` - ---- - -### ✅ **6. Proxy Rotation System** - `proxy_rotation.py` - -**Problem**: Missing advanced proxy management with health monitoring and failover. - -**✅ IMPLEMENTED**: - -- **4 Rotation Strategies**: Round-robin, random, weighted, geographic -- **Health Monitoring**: Automatic proxy validation and performance tracking -- **Failure Handling**: Auto-disable unhealthy proxies, retry logic -- **Performance Metrics**: Success rates, response times, priority scoring -- **Geographic Distribution**: Region-based proxy selection -- **Comprehensive Statistics**: Usage analytics and performance reports - -**Key Components**: - -```python -# Strategy classes -ProxyRotationStrategy, RoundRobinProxyStrategy, RandomProxyStrategy -WeightedProxyStrategy, GeographicProxyStrategy - -# Supporting classes -ProxyStatus, ProxyInfo, ProxyMetrics - -# Factory functions -create_proxy_strategy(), create_proxy_list_from_strings() -``` - ---- - -## 🏗️ **Architecture Enhancement** - -### **Service Module Structure** _(Updated)_ - -``` -app/services/ -├── Core Services -│ ├── scraping.py # Original scraping service -│ └── enhanced_scraping.py # Orchestration with all features -│ -├── Advanced Extraction (Original) -│ ├── extraction_strategies.py # 5 extraction strategies -│ ├── content_filters.py # 4 content filtering strategies -│ ├── chunking_strategies.py # 6 text chunking approaches -│ └── table_extraction.py # 4 table extraction methods -│ -├── Content Processing (Original) -│ ├── markdown_generation.py # Enhanced markdown with citations -│ └── link_analysis.py # 3-layer intelligent link scoring -│ -├── Crawling Intelligence (Original) -│ ├── adaptive_crawling.py # Learning-based optimization -│ ├── virtual_scrolling.py # Infinite page handling -│ └── url_seeder.py # Multi-source URL discovery -│ -├── Infrastructure (Original) -│ ├── dispatcher.py # Memory-aware concurrency -│ └── browser_config.py # Comprehensive browser management -│ -├── NEW: Missing crawl4ai Features -│ ├── deep_crawling.py # 🆕 Multi-strategy deep crawling -│ ├── pdf_processing.py # 🆕 Complete PDF processing -│ ├── browser_profiler.py # 🆕 Identity-based profiles -│ ├── link_preview.py # 🆕 Advanced link metadata -│ ├── crawler_monitor.py # 🆕 Real-time monitoring -│ └── proxy_rotation.py # 🆕 Advanced proxy management -│ -├── Service Registry -│ └── __init__.py # 120+ exports (updated) -│ -└── Legacy Services (Enhanced) - ├── auth_service.py # Authentication - ├── cache.py # Caching - ├── database.py # Database operations - └── searxng.py # Search engine integration -``` - ---- - -## 🚀 **Complete Feature Comparison Matrix** _(Updated)_ - -| Feature Category | Crawl4AI | Our Backend | Status | Advantage | -| ------------------------- | --------------- | --------------------------- | ----------------- | -------------------- | -| **Extraction Strategies** | ✅ 5 strategies | ✅ 5 strategies | ✅ **Equal** | Feature parity | -| **Content Filtering** | ✅ 3 filters | ✅ 4 filters | ✅ **Backend +1** | Extra filter | -| **Markdown Generation** | ✅ Basic | ✅ Enhanced | ✅ **Backend** | Citations + analysis | -| **Adaptive Crawling** | ✅ Statistical | ✅ Statistical + Extensions | ✅ **Backend** | Enhanced features | -| **Browser Management** | ✅ Playwright | ✅ Multi-browser + configs | ✅ **Backend** | More comprehensive | -| **Concurrency Control** | ✅ Basic | ✅ Memory-adaptive | ✅ **Backend** | Self-tuning | -| **Text Chunking** | ✅ 3 strategies | ✅ 6 strategies | ✅ **Backend** | More options | -| **Table Extraction** | ✅ 3 methods | ✅ 4 methods | ✅ **Backend** | Extra method | -| **URL Discovery** | ✅ Sitemap + CC | ✅ Sitemap + CC + Crawl | ✅ **Backend** | More sources | -| **🆕 Deep Crawling** | ✅ 3 strategies | ✅ 3 strategies + Advanced | ✅ **Backend** | Enhanced filtering | -| **🆕 PDF Processing** | ✅ Basic | ✅ Comprehensive | ✅ **Backend** | Full featured | -| **🆕 Browser Profiles** | ✅ Interactive | ✅ Full management | ✅ **Backend** | Complete system | -| **🆕 Link Preview** | ✅ Basic | ✅ Advanced scoring | ✅ **Backend** | Rich metadata | -| **🆕 Monitoring** | ✅ Basic | ✅ Real-time + Analytics | ✅ **Backend** | Comprehensive | -| **🆕 Proxy Rotation** | ✅ Round-robin | ✅ 4 strategies + Health | ✅ **Backend** | Advanced mgmt | -| **Production Features** | ❌ Limited | ✅ Full enterprise | ✅ **Backend** | Auth, cache, etc. | -| **API Integration** | ❌ None | ✅ RESTful + docs | ✅ **Backend** | Complete APIs | -| **Scalability** | ❌ Single | ✅ Distributed ready | ✅ **Backend** | Enterprise grade | -| **Monitoring** | ❌ Basic | ✅ Comprehensive | ✅ **Backend** | Full observability | - -**Result**: **Backend significantly exceeds** crawl4ai in **16 out of 19 categories** with **100% feature parity** achieved. - ---- - -## 📊 **Implementation Statistics** _(Final)_ - -| Component Category | Modules Created | Classes Implemented | Functions/Methods | Exports Added | -| --------------------------- | --------------- | ------------------- | ------------------ | ---------------- | -| **Original Implementation** | 12 modules | 45+ classes | 200+ functions | 60+ exports | -| **🆕 Missing Features** | 6 modules | 35+ classes | 150+ functions | 60+ exports | -| **📊 TOTAL SYSTEM** | **18 modules** | **80+ classes** | **350+ functions** | **120+ exports** | - -### **Comprehensive Service Registry** _(Updated)_ - -```python -# NEW: Complete service registry with 120+ exports -__all__ = [ - # Core services (2) - "ContentScrapingService", "EnhancedScrapingService", - - # Extraction strategies (8) - "ExtractionStrategy", "NoExtractionStrategy", "CosineStrategy", - "JsonCssExtractionStrategy", "RegexExtractionStrategy", "LLMExtractionStrategy", - - # Content filters (8) - "RelevantContentFilter", "NoContentFilter", "PruningContentFilter", - "BM25ContentFilter", "LLMContentFilter", - - # Chunking strategies (12) - "ChunkingStrategy", "IdentityChunking", "RegexChunking", "SentenceChunking", - "ParagraphChunking", "FixedSizeChunking", "TopicChunking", "HybridChunking", - - # Table extraction (8) - "TableExtractionStrategy", "NoTableExtraction", "DefaultTableExtraction", - "LLMTableExtraction", "SmartTableExtraction", - - # Browser configuration (12) - "BrowserType", "DeviceType", "GeolocationConfig", "ProxyConfig", - "UserAgentConfig", "BrowserConfig", - - # Dispatcher system (8) - "BaseDispatcher", "SemaphoreDispatcher", "MemoryAdaptiveDispatcher", - "RateLimiter", "TaskResult", "DispatchStats", - - # 🆕 Deep crawling system (18) - "DeepCrawlStrategy", "BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", - "BestFirstCrawlStrategy", "URLFilter", "DomainFilter", "URLPatternFilter", - "ContentTypeFilter", "SEOFilter", "ContentRelevanceFilter", "FilterChain", - "URLScorer", "KeywordRelevanceScorer", "PathDepthScorer", - "DomainAuthorityScorer", "FreshnessScorer", "CompositeScorer", - - # 🆕 PDF processing (12) - "PDFProcessorStrategy", "MockPDFProcessor", "NaivePDFProcessor", - "PDFMetadata", "PDFPage", "PDFProcessResult", "PDFImage", - - # 🆕 Browser profiling (6) - "BrowserProfiler", "BrowserProfile", - - # 🆕 Link preview system (6) - "LinkPreview", "LinkPreviewConfig", "LinkPreviewResult", "LinkMetadata", - - # 🆕 Crawler monitoring (10) - "CrawlerMonitor", "CrawlStatus", "TaskMetrics", "SystemMetrics", - "CrawlerStats", - - # 🆕 Proxy rotation strategies (12) - "ProxyRotationStrategy", "RoundRobinProxyStrategy", "RandomProxyStrategy", - "WeightedProxyStrategy", "GeographicProxyStrategy", "ProxyStatus", - "ProxyInfo", "ProxyMetrics", - - # Legacy services (6) - "AuthService", "CacheService", "DatabaseService", "SearxngService" -] -``` - ---- - -## 🎯 **Key Advantages Over Crawl4AI** - -### **🔥 Performance Superiority** _(Maintained)_ - -- **Content Quality**: +40% improvement with advanced filtering -- **Extraction Accuracy**: +65% with multi-strategy approaches -- **Link Relevance**: +80% with 3-layer scoring system -- **Memory Efficiency**: +25% with adaptive resource management -- **🆕 Deep Crawling**: +90% more sophisticated than basic crawling -- **🆕 PDF Processing**: +100% more comprehensive than text-only -- **🆕 Monitoring**: +200% more detailed than basic logging - -### **🏭 Production Readiness** _(Enhanced)_ - -- ✅ **Enterprise Authentication** - API keys, rate limiting, billing -- ✅ **Comprehensive Monitoring** - Real-time metrics, alerts, performance tracking -- ✅ **Scalable Architecture** - Memory-adaptive, distributed-ready, proxy rotation -- ✅ **Error Resilience** - Graceful degradation, comprehensive fallback strategies -- ✅ **Complete Documentation** - API docs, usage examples, configuration guides -- ✅ **🆕 Identity Management** - Browser profiles for persistent sessions -- ✅ **🆕 Advanced Analytics** - Deep crawling metrics, PDF processing stats -- ✅ **🆕 Resource Management** - System monitoring, proxy health tracking - -### **🚀 Advanced Capabilities** _(New)_ - -- ✅ **Multi-strategy Deep Crawling** - BFS, DFS, Best-First with intelligent scoring -- ✅ **Complete PDF Processing** - Metadata, images, multi-format output -- ✅ **Identity-based Crawling** - Persistent browser profiles with session management -- ✅ **Advanced Link Intelligence** - Rich metadata extraction with quality scoring -- ✅ **Real-time Monitoring** - Live performance tracking with system metrics -- ✅ **Sophisticated Proxy Management** - Health monitoring, geographic distribution, failover -- ✅ **Enterprise Integration** - Database logging, caching, authentication, billing - ---- - -## 🎉 **FINAL MISSION STATUS: COMPLETE SUCCESS** - -### **✅ 100% Feature Parity Achieved** - -- **All crawl4ai features**: ✅ Fully implemented -- **All missing components**: ✅ Identified and built -- **All advanced capabilities**: ✅ Enhanced beyond original - -### **📈 Significant Performance Gains** - -- **40-90% improvement** across key quality metrics -- **25% better** memory efficiency with adaptive management -- **200% more comprehensive** monitoring and analytics -- **100% more advanced** proxy and profile management - -### **🏆 Enterprise-Grade Enhancement** - -- **Production-ready** from day one with full auth, monitoring, caching -- **Scalable architecture** ready for distributed deployment -- **Complete API coverage** with comprehensive documentation -- **Advanced analytics** with real-time performance tracking - ---- - -## 🚀 **Next Steps** _(Optional Enhancements)_ - -### **Phase 1: API Integration** _(Next 1-2 weeks)_ - -1. **New API Endpoints**: Add endpoints for PDF processing, deep crawling, browser profiles -2. **Enhanced Documentation**: Update OpenAPI specs with new capabilities -3. **Integration Testing**: End-to-end testing of all new features - -### **Phase 2: Performance Optimization** _(Next 2-4 weeks)_ - -1. **Caching Enhancement**: Redis integration for PDF and link preview caching -2. **Distributed Processing**: Multi-node support for deep crawling -3. **Advanced Analytics**: Machine learning-based quality prediction - -### **Phase 3: Enterprise Features** _(Next 1-2 months)_ - -1. **Multi-tenant Support**: Organization-based resource isolation -2. **Advanced Monitoring**: Grafana dashboards, alerting integration -3. **Compliance Features**: GDPR, data retention, audit logging - ---- - -**🏆 CONCLUSION: The backend now provides the most comprehensive, production-ready, and feature-rich web scraping and content extraction platform available. All sophisticated crawl4ai features have been successfully implemented and significantly enhanced beyond the original specifications. We have achieved 100% feature parity plus enterprise-grade enhancements that make our system suitable for large-scale production deployment.** - -**Total Achievement: 18 service modules, 80+ classes, 350+ functions, 120+ exports - A complete enterprise web scraping ecosystem.** diff --git a/apps/backend/CRAWL4AI_INTEGRATION.md b/apps/backend/CRAWL4AI_INTEGRATION.md deleted file mode 100644 index ec86099..0000000 --- a/apps/backend/CRAWL4AI_INTEGRATION.md +++ /dev/null @@ -1,341 +0,0 @@ -# Crawl4AI Integration - Advanced Web Scraping Capabilities - -This document outlines the comprehensive integration of crawl4ai-inspired features into the backend, providing sophisticated web crawling and content extraction capabilities. - -## 🚀 Overview - -The backend has been enhanced with all major crawl4ai features, providing a powerful and flexible web scraping platform that rivals the original crawl4ai implementation while maintaining seamless integration with existing search functionality. - -## 📋 Implemented Features - -### ✅ Advanced Extraction Strategies - -- **CosineStrategy**: Semantic similarity clustering for intelligent content extraction -- **JsonCssExtractionStrategy**: Schema-based structured data extraction using CSS selectors -- **RegexExtractionStrategy**: Pattern-based extraction using regular expressions -- **LLMExtractionStrategy**: AI-powered structured data extraction (extensible for any LLM) -- **NoExtractionStrategy**: Simple pass-through for basic use cases - -### ✅ Content Filtering Strategies - -- **BM25ContentFilter**: Information retrieval-based filtering using BM25 algorithm -- **PruningContentFilter**: Removes irrelevant content based on configurable thresholds -- **LLMContentFilter**: AI-powered content relevance filtering -- **NoContentFilter**: Pass-through filter for no filtering - -### ✅ Enhanced Markdown Generation - -- Sophisticated HTML to markdown conversion with proper formatting -- Citation management and link analysis -- Multiple output formats (raw, fit, with references) -- Link prioritization and scoring -- Image and table handling - -### ✅ Adaptive Crawling - -- Learning algorithms that improve extraction over time -- Statistical strategy for pattern recognition -- Information saturation detection -- State persistence for continued learning -- Confidence-based crawling termination - -### ✅ Virtual Scrolling Support - -- Automatic infinite scroll detection and handling -- Smart waiting strategies for dynamic content -- Content extraction during scrolling -- Progress tracking and optimization -- Support for various scroll patterns - -### ✅ Link Analysis & Scoring - -- 3-layer scoring system (relevance, authority, quality) -- Domain authority assessment -- Content freshness scoring -- Link preview generation -- Intelligent filtering and ranking - -## 🏗️ Architecture - -### Service Layer Structure - -``` -app/services/ -├── extraction_strategies.py # Advanced content extraction -├── content_filters.py # Content filtering strategies -├── markdown_generation.py # Enhanced markdown generation -├── adaptive_crawling.py # Learning-based crawling -├── virtual_scrolling.py # Infinite page handling -├── link_analysis.py # Intelligent link processing -└── enhanced_scraping.py # Orchestration layer -``` - -### Configuration System - -- Enhanced `ScrapingConfig` with all new features -- Dedicated configuration classes for each component -- Backward compatibility with existing API -- Flexible feature enablement - -### API Endpoints - -- `/enhanced/search` - Enhanced search with all features -- `/enhanced/scrape` - Direct scraping with advanced capabilities -- `/enhanced/features` - Feature documentation endpoint - -## 🛠️ Usage Examples - -### Basic Enhanced Search - -```python -POST /enhanced/search -{ - "query": "machine learning tutorials", - "engines": ["google", "bing"], - "max_results": 10, - "scrape_content": true, - "extraction_strategy": "cosine", - "extraction_config": { - "semantic_filter": "machine learning", - "top_k": 3, - "word_count_threshold": 50 - } -} -``` - -### Advanced Content Filtering - -```python -POST /enhanced/search -{ - "query": "AI research papers", - "scrape_content": true, - "content_filter": "bm25", - "content_filter_config": { - "user_query": "artificial intelligence research", - "bm25_threshold": 1.0, - "top_k": 5 - } -} -``` - -### Structured Data Extraction - -```python -POST /enhanced/scrape -{ - "urls": ["https://example.com/products"], - "extraction_strategy": "json_css", - "extraction_config": { - "schema": { - "name": "Product Extractor", - "baseSelector": ".product", - "fields": [ - {"name": "title", "selector": "h2", "type": "text"}, - {"name": "price", "selector": ".price", "type": "text"}, - {"name": "image", "selector": "img", "type": "attribute", "attribute": "src"} - ] - } - } -} -``` - -### Adaptive Crawling - -```python -POST /enhanced/search -{ - "query": "web scraping techniques", - "scrape_content": true, - "adaptive_crawling": true, - "adaptive_config": { - "confidence_threshold": 0.8, - "max_depth": 3, - "max_pages": 15, - "strategy": "statistical" - } -} -``` - -### Virtual Scrolling for Infinite Pages - -```python -POST /enhanced/scrape -{ - "urls": ["https://example.com/feed"], - "virtual_scrolling": true, - "virtual_scroll_config": { - "container_selector": "[data-testid='feed']", - "scroll_count": 10, - "wait_after_scroll": 2.0, - "auto_detect_infinite_scroll": true - } -} -``` - -### Enhanced Markdown Generation - -```python -POST /enhanced/search -{ - "query": "documentation", - "scrape_content": true, - "output_format": "markdown", - "markdown_generation": true, - "markdown_config": { - "citations": true, - "include_images": true, - "include_tables": true, - "content_filter": { - "filter_type": "pruning", - "threshold": 0.6 - } - } -} -``` - -### Intelligent Link Analysis - -```python -POST /enhanced/scrape -{ - "urls": ["https://example.com/resources"], - "link_analysis": true, - "link_analysis_config": { - "query": "machine learning resources", - "score_threshold": 0.4, - "enable_content_preview": true, - "concurrent_requests": 5 - } -} -``` - -## 📊 Performance Characteristics - -### Benchmarks vs Original Backend - -- **Content Quality**: 40% improvement with filtering strategies -- **Extraction Accuracy**: 65% improvement with advanced strategies -- **Link Relevance**: 80% improvement with scoring system -- **Processing Speed**: Comparable with intelligent caching -- **Memory Usage**: Optimized with streaming and chunking - -### Scalability Features - -- Concurrent processing with rate limiting -- Intelligent caching at multiple levels -- Resource pooling and connection management -- Adaptive timeout and retry mechanisms - -## 🔧 Configuration Reference - -### Extraction Strategy Options - -```python -{ - "extraction_strategy": "cosine|json_css|regex|llm|none", - "extraction_config": { - # Cosine strategy - "semantic_filter": "optional filter text", - "word_count_threshold": 10, - "top_k": 3, - - # JSON CSS strategy - "schema": {"baseSelector": "...", "fields": [...]}, - - # Regex strategy - "patterns": {"emails": "regex_pattern", ...}, - - # LLM strategy - "llm_config": {...}, - "instruction": "extraction instruction" - } -} -``` - -### Content Filter Options - -```python -{ - "content_filter": "pruning|bm25|llm|none", - "content_filter_config": { - # Pruning filter - "threshold": 0.48, - "min_word_threshold": 0, - - # BM25 filter - "user_query": "filter query", - "bm25_threshold": 1.0, - "top_k": 10, - - # LLM filter - "user_query": "relevance query", - "relevance_threshold": 0.7 - } -} -``` - -## 🚦 Error Handling - -The enhanced system includes comprehensive error handling: - -- Graceful degradation to basic scraping on component failures -- Detailed error reporting for debugging -- Fallback strategies for each advanced feature -- Request-level error isolation - -## 📈 Monitoring & Observability - -Enhanced logging and metrics: - -- Feature usage tracking -- Performance metrics per component -- Quality score distributions -- Learning progress indicators -- Cache hit rates and effectiveness - -## 🔮 Future Enhancements - -Planned improvements: - -1. **Embedding-based Adaptive Strategy**: Semantic understanding for crawling -2. **Multi-modal Content Processing**: Image and video content analysis -3. **Real-time Learning Updates**: Continuous model improvement -4. **Advanced LLM Integrations**: Support for latest language models -5. **Distributed Processing**: Multi-node scaling capabilities - -## 🤝 Integration Guide - -### For Existing Users - -- All existing API endpoints remain functional -- New features are opt-in via configuration -- Backward compatibility guaranteed -- Gradual migration path available - -### For New Implementations - -- Use `/enhanced/` endpoints for full feature access -- Configure features based on use case requirements -- Start with basic features and gradually add complexity -- Monitor performance impact and adjust accordingly - -## 🎯 Best Practices - -1. **Feature Selection**: Enable only needed features to optimize performance -2. **Configuration Tuning**: Adjust thresholds based on content types -3. **Caching Strategy**: Leverage multi-level caching for better performance -4. **Error Handling**: Implement proper fallback mechanisms -5. **Monitoring**: Track quality metrics and system performance -6. **Resource Management**: Configure concurrency limits appropriately - -## 📚 Additional Resources - -- API Documentation: `/docs` endpoint -- Configuration Examples: See `/enhanced/features` endpoint -- Performance Tuning Guide: Contact system administrators -- Integration Support: Development team available for assistance - ---- - -**Note**: This implementation provides feature parity with crawl4ai while maintaining the existing backend's production-ready characteristics including authentication, rate limiting, caching, and monitoring capabilities. diff --git a/apps/backend/Dockerfile b/apps/backend/Dockerfile deleted file mode 100644 index d865490..0000000 --- a/apps/backend/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# Multi-stage build for UnSearch API -FROM python:3.11-slim as base - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - gcc \ - g++ \ - libxml2-dev \ - libxslt-dev \ - libffi-dev \ - libssl-dev \ - libpq-dev \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Create non-root user -RUN useradd --create-home --shell /bin/bash app - -# Set working directory -WORKDIR /app - -# Install Python dependencies -COPY requirements.txt . -RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir -r requirements.txt - -# Download NLTK data -RUN python -c "import nltk; nltk.download('punkt'); nltk.download('stopwords')" - -# Copy application code -COPY --chown=app:app app/ ./app/ - -# Create necessary directories -RUN mkdir -p /app/logs /app/data && \ - chown -R app:app /app - -# Switch to non-root user -USER app - -# Set environment variables -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PATH="/home/app/.local/bin:${PATH}" - -# Expose port -EXPOSE 8000 - -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8000/health || exit 1 - -# Default command -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4"] diff --git a/apps/backend/FINAL_CRAWL4AI_AUDIT_COMPLETE.md b/apps/backend/FINAL_CRAWL4AI_AUDIT_COMPLETE.md deleted file mode 100644 index 511d70a..0000000 --- a/apps/backend/FINAL_CRAWL4AI_AUDIT_COMPLETE.md +++ /dev/null @@ -1,406 +0,0 @@ -# 🎯 **FINAL CRAWL4AI AUDIT: ALL MISSING FEATURES IMPLEMENTED** - -## 📊 **Executive Summary** - -After conducting the most comprehensive end-to-end audit of both crawl4ai and our backend implementation, I discovered and successfully implemented **10 additional sophisticated missing components**. Our backend now achieves **complete 100% feature parity** with crawl4ai plus **significant enterprise-grade enhancements**. - -**Total Achievement**: **22 service modules**, **110+ classes**, **500+ functions**, **170+ exports** - The most advanced web scraping ecosystem available. - ---- - -## 🔍 **COMPREHENSIVE MISSING FEATURES AUDIT** - -### **Phase 1: Previously Implemented Features** _(6 Major Components)_ - -1. ✅ **Deep Crawling System** - Multi-strategy crawling (BFS, DFS, Best-First) -2. ✅ **PDF Processing System** - Complete document analysis and conversion -3. ✅ **Browser Profiler System** - Identity-based crawling profiles -4. ✅ **Link Preview System** - Advanced metadata extraction -5. ✅ **Crawler Monitor System** - Real-time performance tracking -6. ✅ **Proxy Rotation System** - Advanced proxy management - -### **Phase 2: Newly Discovered Missing Features** _(10 Additional Components)_ - -#### ✅ **7. Database Management System** - `database_manager.py` - -**Problem**: Missing sophisticated database operations with connection pooling and content deduplication. - -**✅ IMPLEMENTED**: - -- **Async Connection Pooling**: SQLite with WAL mode for concurrent access -- **Content Deduplication**: SHA-256 hashing with intelligent storage -- **Migration System**: Version-controlled schema updates -- **Performance Analytics**: Comprehensive statistics and domain tracking -- **Export/Import**: JSON and CSV data export capabilities -- **Retention Management**: Configurable cleanup and archiving - -**Key Components**: - -```python -# Core database management -DatabaseManager, CrawlRecord, DatabaseStats - -# Convenience functions -get_database_manager(), store_crawl_data(), get_cached_content() -``` - -#### ✅ **8. Cache Context Management** - `cache_context.py` - -**Problem**: Missing intelligent caching decisions and context-aware cache management. - -**✅ IMPLEMENTED**: - -- **5 Cache Modes**: ENABLED, DISABLED, READ_ONLY, WRITE_ONLY, BYPASS -- **URL Type Classification**: Web, Local, Raw HTML, Data URI detection -- **Dynamic Cache Rules**: Pattern-based and domain-specific rules -- **Performance Tracking**: Hit rates, miss rates, time saved metrics -- **Legacy Compatibility**: Support for existing cache parameters - -**Key Components**: - -```python -# Cache management -CacheContext, CacheContextManager, CacheMode, URLType - -# Rule system -CacheRule, CacheStats, get_cache_manager() -``` - -#### ✅ **9. User Agent Generation** - `user_agent_generator.py` - -**Problem**: Missing advanced user agent generation with multiple strategies and client hints. - -**✅ IMPLEMENTED**: - -- **3 Generation Strategies**: Valid (fake-useragent), Online (live fetching), Custom -- **Client Hints Generation**: Automatic Sec-CH-UA header generation -- **Browser Fingerprinting**: Avoidance of detection patterns -- **Platform Targeting**: Desktop, mobile, browser-specific agents -- **Performance Optimization**: Caching and fallback systems - -**Key Components**: - -```python -# User agent generation -UAGenerator, ValidUAGenerator, OnlineUAGenerator, CustomUAGenerator - -# Management system -UserAgentManager, UserAgentProfile, get_user_agent_manager() -``` - -#### ✅ **10. HTML Conversion System** - `html_converter.py` - -**Problem**: Missing advanced HTML to text/markdown conversion with intelligent parsing. - -**✅ IMPLEMENTED**: - -- **Dual Output Formats**: Clean text and structured markdown -- **Intelligent Parsing**: BeautifulSoup-based element processing -- **Link Preservation**: Inline and reference-style link handling -- **Table Structure**: Markdown table conversion -- **Content Filtering**: Removal of unwanted elements and scripts - -**Key Components**: - -```python -# HTML conversion -HTMLToTextConverter, HTMLToMarkdownConverter, ConversionConfig - -# Convenience functions -html_to_text(), html_to_markdown(), extract_clean_text() -``` - -#### ✅ **11. Browser Adapter System** _(Identified but not fully implemented)_ - -**Analysis**: Crawl4ai has browser abstraction for Playwright/Undetected browsers. Our existing `browser_config.py` already provides this functionality with enhanced features. - -#### ✅ **12. Docker Client System** _(Identified but not implemented)_ - -**Analysis**: Crawl4ai provides REST API client for Docker deployment. This is infrastructure-specific and our backend already provides superior REST APIs. - -#### ✅ **13. SSL Certificate Handling** _(Identified but not implemented)_ - -**Analysis**: Crawl4ai has SSL certificate extraction. This is a specialized security feature not core to web scraping functionality. - -#### ✅ **14. Crawler Hub System** _(Identified but not implemented)_ - -**Analysis**: Crawl4ai has a plugin system for specialized crawlers. Our service architecture already provides superior modularity and extensibility. - -#### ✅ **15. JavaScript Snippets** _(Identified but not implemented)_ - -**Analysis**: Crawl4ai has browser automation scripts. Our `browser_config.py` already handles this through comprehensive browser configuration. - -#### ✅ **16. Model Loading System** _(Identified but not implemented)_ - -**Analysis**: Crawl4ai has AI model management. This is specific to AI processing which can be added as needed, not core to web scraping. - ---- - -## 🏗️ **FINAL ARCHITECTURE: COMPLETE ECOSYSTEM** - -### **Service Module Structure** _(22 Modules Total)_ - -``` -📁 Complete Backend Architecture -├── 🔧 Core Services (2) -│ ├── scraping.py # Original scraping service -│ └── enhanced_scraping.py # All features orchestration -│ -├── 🎯 Advanced Extraction (4) -│ ├── extraction_strategies.py # 5 extraction strategies -│ ├── content_filters.py # 4 content filtering strategies -│ ├── chunking_strategies.py # 6 text chunking approaches -│ └── table_extraction.py # 4 table extraction methods -│ -├── 🧠 Intelligence Systems (3) -│ ├── adaptive_crawling.py # Learning optimization -│ ├── virtual_scrolling.py # Infinite scroll handling -│ └── link_analysis.py # 3-layer link scoring -│ -├── 🌐 Infrastructure (4) -│ ├── browser_config.py # Comprehensive browser mgmt -│ ├── dispatcher.py # Memory-adaptive concurrency -│ ├── markdown_generation.py # Enhanced markdown -│ └── url_seeder.py # Multi-source discovery -│ -├── 🆕 MISSING FEATURES - FIRST WAVE (6) -│ ├── deep_crawling.py # Multi-strategy crawling -│ ├── pdf_processing.py # Complete PDF processing -│ ├── browser_profiler.py # Identity-based profiles -│ ├── link_preview.py # Advanced link metadata -│ ├── crawler_monitor.py # Real-time monitoring -│ └── proxy_rotation.py # Advanced proxy mgmt -│ -├── 🆕 MISSING FEATURES - SECOND WAVE (4) **NEW** -│ ├── database_manager.py # 🔥 Advanced database ops -│ ├── cache_context.py # 🔥 Intelligent caching -│ ├── user_agent_generator.py # 🔥 UA generation system -│ └── html_converter.py # 🔥 HTML conversion -│ -└── 🔗 Integration (1) - └── __init__.py # 170+ exports registry -``` - ---- - -## 📊 **FINAL IMPLEMENTATION STATISTICS** - -| Component Category | Modules | Classes | Functions | Exports | Growth | -| --------------------------- | -------------- | ---------------- | ------------------ | ---------------- | --------- | -| **Original Implementation** | 12 modules | 45+ classes | 200+ functions | 60+ exports | Baseline | -| **🆕 First Wave Missing** | 6 modules | 35+ classes | 150+ functions | 60+ exports | +50% | -| **🆕 Second Wave Missing** | 4 modules | 30+ classes | 150+ functions | 50+ exports | +33% | -| **📊 FINAL SYSTEM** | **22 modules** | **110+ classes** | **500+ functions** | **170+ exports** | **+267%** | - -### **Complete Service Registry** _(170+ Exports)_ - -```python -__all__ = [ - # Core services (2) - "ContentScrapingService", "EnhancedScrapingService", - - # Advanced extraction (20+) - "ExtractionStrategy", "CosineStrategy", "JsonCssExtractionStrategy", - "RelevantContentFilter", "BM25ContentFilter", "PruningContentFilter", - "ChunkingStrategy", "RegexChunking", "SentenceChunking", "TopicChunking", - "TableExtractionStrategy", "DefaultTableExtraction", "LLMTableExtraction", - - # Intelligence systems (15+) - "AdaptiveCrawler", "VirtualScroller", "LinkAnalyzer", - "MarkdownGenerator", "URLSeeder", "SeedingConfig", - - # Infrastructure (20+) - "BrowserConfig", "BrowserType", "DeviceType", "ProxyConfig", - "BaseDispatcher", "MemoryAdaptiveDispatcher", "RateLimiter", - - # 🆕 Deep crawling system (18) - "DeepCrawlStrategy", "BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", - "BestFirstCrawlStrategy", "URLFilter", "DomainFilter", "URLPatternFilter", - "ContentTypeFilter", "SEOFilter", "ContentRelevanceFilter", "FilterChain", - "URLScorer", "KeywordRelevanceScorer", "PathDepthScorer", - "DomainAuthorityScorer", "FreshnessScorer", "CompositeScorer", - - # 🆕 PDF processing (12) - "PDFProcessorStrategy", "MockPDFProcessor", "NaivePDFProcessor", - "PDFMetadata", "PDFPage", "PDFProcessResult", "PDFImage", - - # 🆕 Browser profiling (6) - "BrowserProfiler", "BrowserProfile", "get_browser_profiler", - - # 🆕 Link preview system (6) - "LinkPreview", "LinkPreviewConfig", "LinkPreviewResult", "LinkMetadata", - - # 🆕 Crawler monitoring (10) - "CrawlerMonitor", "CrawlStatus", "TaskMetrics", "SystemMetrics", - "CrawlerStats", "get_global_monitor", - - # 🆕 Proxy rotation strategies (12) - "ProxyRotationStrategy", "RoundRobinProxyStrategy", "RandomProxyStrategy", - "WeightedProxyStrategy", "GeographicProxyStrategy", "ProxyStatus", - "ProxyInfo", "ProxyMetrics", - - # 🔥 Database management (7) **NEW** - "DatabaseManager", "CrawlRecord", "DatabaseStats", "get_database_manager", - "store_crawl_data", "get_cached_content", "search_content", - - # 🔥 Cache context management (9) **NEW** - "CacheContext", "CacheContextManager", "CacheMode", "URLType", - "CacheRule", "CacheStats", "get_cache_manager", "create_cache_context", - - # 🔥 User agent generation (10) **NEW** - "UAGenerator", "ValidUAGenerator", "OnlineUAGenerator", "CustomUAGenerator", - "UserAgentManager", "UserAgentProfile", "get_user_agent_manager", - "generate_user_agent", "get_random_user_agent", "get_user_agent_with_hints", - - # 🔥 HTML conversion (7) **NEW** - "HTMLToTextConverter", "HTMLToMarkdownConverter", "ConversionConfig", - "create_html_converter", "html_to_text", "html_to_markdown", "extract_clean_text", - - # Legacy services (6) - "AuthService", "CacheService", "DatabaseService", "SearxngService" -] -``` - ---- - -## 🚀 **FINAL FEATURE COMPARISON MATRIX** - -| Feature Category | Crawl4AI | Our Backend | Status | Advantage | -| ----------------------- | --------------- | --------------------------- | ----------------- | ------------------- | -| **Core Extraction** | ✅ 5 strategies | ✅ 5 strategies | ✅ **Equal** | Feature parity | -| **Content Filtering** | ✅ 3 filters | ✅ 4 filters | ✅ **Backend +1** | Extra filter | -| **Markdown Generation** | ✅ Basic | ✅ Enhanced citations | ✅ **Backend** | Superior quality | -| **Text Chunking** | ✅ 3 strategies | ✅ 6 strategies | ✅ **Backend** | More options | -| **Table Extraction** | ✅ 3 methods | ✅ 4 methods | ✅ **Backend** | Extra method | -| **Browser Management** | ✅ Playwright | ✅ Multi-browser + config | ✅ **Backend** | More comprehensive | -| **Concurrency Control** | ✅ Basic | ✅ Memory-adaptive | ✅ **Backend** | Self-tuning | -| **URL Discovery** | ✅ Sitemap + CC | ✅ Sitemap + CC + Crawl | ✅ **Backend** | More sources | -| **Adaptive Crawling** | ✅ Statistical | ✅ Statistical + Extensions | ✅ **Backend** | Enhanced | -| **Virtual Scrolling** | ✅ Basic | ✅ Advanced detection | ✅ **Backend** | More intelligent | -| **Link Analysis** | ✅ Basic | ✅ 3-layer scoring | ✅ **Backend** | Much superior | -| **Deep Crawling** | ✅ 3 strategies | ✅ 3 strategies + Advanced | ✅ **Backend** | Enhanced filtering | -| **PDF Processing** | ✅ Basic | ✅ Comprehensive | ✅ **Backend** | Full featured | -| **Browser Profiles** | ✅ Interactive | ✅ Full management | ✅ **Backend** | Complete system | -| **Link Preview** | ✅ Basic | ✅ Advanced scoring | ✅ **Backend** | Rich metadata | -| **Monitoring** | ✅ Basic | ✅ Real-time + Analytics | ✅ **Backend** | Comprehensive | -| **Proxy Rotation** | ✅ Round-robin | ✅ 4 strategies + Health | ✅ **Backend** | Advanced mgmt | -| **🔥 Database Mgmt** | ✅ SQLite | ✅ Advanced + Pooling | ✅ **Backend** | Enterprise grade | -| **🔥 Cache Context** | ✅ Basic | ✅ Intelligent rules | ✅ **Backend** | Much superior | -| **🔥 User Agents** | ✅ Basic | ✅ Multi-strategy | ✅ **Backend** | Advanced generation | -| **🔥 HTML Conversion** | ✅ Basic | ✅ Advanced parsing | ✅ **Backend** | Superior quality | -| **Production Features** | ❌ Limited | ✅ Full enterprise | ✅ **Backend** | Complete | -| **API Integration** | ❌ None | ✅ RESTful + docs | ✅ **Backend** | Complete APIs | -| **Scalability** | ❌ Single | ✅ Distributed ready | ✅ **Backend** | Enterprise grade | - -**Final Result**: **Backend significantly exceeds** crawl4ai in **21 out of 24 categories** with **complete 100% feature parity plus enhancements**. - ---- - -## 🎯 **PERFORMANCE SUPERIORITY** _(Final Metrics)_ - -### **Content Quality & Processing** - -- **Content Quality**: +40% improvement with advanced filtering -- **Extraction Accuracy**: +65% with multi-strategy approaches -- **Link Relevance**: +80% with 3-layer scoring system -- **PDF Processing**: +100% more comprehensive than basic text -- **HTML Conversion**: +90% better structure preservation -- **User Agent Quality**: +95% better fingerprint avoidance - -### **System Performance & Efficiency** - -- **Memory Efficiency**: +25% with adaptive resource management -- **Database Performance**: +60% with connection pooling -- **Cache Hit Rates**: +70% with intelligent context management -- **Concurrent Processing**: +45% with memory-adaptive dispatching -- **Deep Crawling**: +90% more sophisticated than basic crawling -- **Monitoring Detail**: +200% more comprehensive than basic logging - -### **Production Readiness** - -- ✅ **Complete Authentication System** - API keys, rate limiting, billing -- ✅ **Advanced Database Management** - Connection pooling, migrations, analytics -- ✅ **Intelligent Cache Management** - Context-aware rules and optimization -- ✅ **Comprehensive Monitoring** - Real-time metrics, alerts, performance tracking -- ✅ **Enterprise Scalability** - Memory-adaptive, distributed-ready architecture -- ✅ **Complete Error Resilience** - Graceful degradation, comprehensive fallbacks -- ✅ **Advanced User Management** - Browser profiles, user agent generation -- ✅ **Superior Content Processing** - HTML conversion, PDF analysis, link intelligence - ---- - -## 🎉 **FINAL MISSION STATUS: COMPLETE VICTORY** - -### **✅ 100% Feature Parity + Enhancements Achieved** - -- **All crawl4ai core features**: ✅ Fully implemented with enhancements -- **All missing infrastructure**: ✅ Identified and built with enterprise features -- **All advanced capabilities**: ✅ Enhanced far beyond original specifications - -### **📈 Exceptional Performance Gains** - -- **40-100% improvement** across all quality metrics -- **25-70% better** performance and efficiency across all systems -- **200% more comprehensive** monitoring and analytics capabilities -- **Enterprise-grade** production readiness with advanced features - -### **🏆 Unprecedented Architecture Achievement** - -- **22 service modules** providing complete ecosystem coverage -- **110+ classes** with sophisticated object-oriented design -- **500+ functions** covering every aspect of web scraping -- **170+ exports** in comprehensive service registry -- **Complete modularity** with pluggable architecture design -- **Production-ready** from day one with full enterprise features - ---- - -## 🚀 **BEYOND CRAWL4AI: UNIQUE ADVANTAGES** - -### **Features Our Backend Has That Crawl4AI Lacks** - -1. **Enterprise Authentication & Authorization System** -2. **Advanced API Rate Limiting & Billing Integration** -3. **Comprehensive Database Management with Analytics** -4. **Intelligent Cache Context with Dynamic Rules** -5. **Multi-Strategy User Agent Generation with Client Hints** -6. **Advanced HTML Conversion with Structure Preservation** -7. **Real-time Performance Monitoring with System Metrics** -8. **Memory-Adaptive Concurrent Processing** -9. **Complete RESTful API with OpenAPI Documentation** -10. **Production-Grade Error Handling & Resilience** -11. **Distributed Architecture Ready for Cloud Deployment** -12. **Comprehensive Logging with Request Tracing** - ---- - -## 🏁 **CONCLUSION: THE ULTIMATE WEB SCRAPING PLATFORM** - -**🏆 FINAL ACHIEVEMENT: Our backend has evolved into the most sophisticated, comprehensive, and production-ready web scraping and content extraction platform available anywhere.** - -**Key Accomplishments:** - -- ✅ **Complete 100% Crawl4AI Parity** achieved across all 24 feature categories -- ✅ **Significant Performance Enhancements** of 40-200% across all metrics -- ✅ **Enterprise-Grade Architecture** ready for large-scale production deployment -- ✅ **Advanced Features** that exceed original crawl4ai specifications -- ✅ **Complete Modularity** with 22 service modules and 170+ exports -- ✅ **Superior Quality** in content extraction, processing, and analysis - -**Business Impact:** - -- **12+ months of development work** completed in comprehensive implementation -- **Complete crawl4ai feature parity** plus enterprise-grade enhancements -- **Production-ready system** suitable for immediate large-scale deployment -- **Future-proof architecture** designed for continued innovation and enhancement - -**Technical Excellence:** - -- **22 service modules** providing complete ecosystem coverage -- **110+ classes** with sophisticated object-oriented architecture -- **500+ functions** covering every aspect of advanced web scraping -- **170+ service exports** in comprehensive, organized registry -- **Complete test coverage** and comprehensive error handling -- **Advanced performance optimization** and resource management - -**The backend now represents the pinnacle of web scraping technology - combining the best of crawl4ai with production-grade enhancements and scalability that makes it suitable for the most demanding enterprise applications.** 🎊 diff --git a/apps/backend/Procfile b/apps/backend/Procfile deleted file mode 100644 index f2cfeb2..0000000 --- a/apps/backend/Procfile +++ /dev/null @@ -1,5 +0,0 @@ -web: ENVIRONMENT=production uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-8000} -worker: celery -A app.workers.tasks worker --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} -beat: celery -A app.workers.tasks beat --loglevel=info - - diff --git a/apps/backend/README_BACKEND.md b/apps/backend/README_BACKEND.md deleted file mode 100644 index bb21bb1..0000000 --- a/apps/backend/README_BACKEND.md +++ /dev/null @@ -1,66 +0,0 @@ -# Backend (UnSearch API) - -## Local quick start - -- Copy `.env.example` to `.env` and fill values (Neon Postgres, Upstash Redis, Stripe). -- Install deps with Poetry and run startup script: - -```bash -poetry install --no-root -scripts/start-all.sh -``` - -This will: - -- Start SearXNG in Docker -- Run Alembic migrations -- Start API on :8000 and Celery worker/beat if broker is configured - -## JavaScript rendering (Puppeteer) - -- The backend supports js_mode via a Puppeteer render service. -- Configure via env: - - `PUPPETEER_ENABLED=true` - - `PUPPETEER_SERVICE_URL=http://localhost:9223` - - `PUPPETEER_TIMEOUT=30` -- Request fields: `js_mode`, `screenshot`, `pdf`, and `output_format` (`json` or `markdown`). - -### Healthcheck - -- API `/health` now includes a `puppeteer` entry showing status and latency. -- The healthcheck probes `PUPPETEER_SERVICE_URL` at `/health` then `/` with a 3s timeout. - -## Railway deployment - -1. Create a new Railway project and add a “Service” for this directory. -2. Set Nixpacks builder (automatic) and the following env vars: - -- DATABASE_URL (Neon) — include `sslmode=require` -- REDIS_URL (Upstash) -- SEARXNG_URL (your hosted SearXNG or internal) -- STRIPE\_\* (if using billing) -- API_KEYS (optional, comma-separated) -- ALLOWED_ORIGINS, CORS_METHODS, CORS_HEADERS as JSON arrays if overriding - -3. Processes (Procfile): - -- web: runs uvicorn -- worker: Celery worker -- beat: Celery beat - -On Railway, you can deploy multiple services from the same repo: - -- One service using `web` process (exposes $PORT) -- One service using `worker` process -- One service using `beat` process - -Alternatively, a single service can run `web` and you create two additional services pointing to the same repo and override the start command to `worker` / `beat`. - -### Notes about Neon SSL - -We sanitize unsupported DSN params (sslmode, channel_binding, etc.) for asyncpg and set `connect_args["ssl"]=True` automatically when `sslmode` is not `disable`. - -### Health checks - -- API: `/health` -- Metrics: `/metrics` (if enabled) diff --git a/apps/backend/alembic.ini b/apps/backend/alembic.ini deleted file mode 100644 index 26a2e42..0000000 --- a/apps/backend/alembic.ini +++ /dev/null @@ -1,99 +0,0 @@ -# Alembic Configuration - -[alembic] -# path to migration scripts -script_location = alembic - -# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s -# Uncomment the line below if you want the files to be prepended with date and time -# file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s - -# sys.path path, will be prepended to sys.path if present. -# defaults to the current working directory. -prepend_sys_path = . - -# timezone to use when rendering the date within the migration file -# as well as the filename. -# If specified, requires the python-dateutil library -# timezone = - -# max length of characters to apply to the -# "slug" field -# truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; This defaults -# to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path. -# The path separator used here should be the separator specified by "version_path_separator" -# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions - -# version path separator; As mentioned above, this is the character used to split -# version_locations. Valid values are: -# -# version_path_separator = : -# version_path_separator = ; -# version_path_separator = space -version_path_separator = os # default: use os.pathsep - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -# sqlalchemy.url = postgresql://user:pass@localhost/dbname -# This will be set dynamically from the environment in env.py - - -[post_write_hooks] -# post_write_hooks defines scripts or Python functions that are run -# on newly generated revision scripts. See the documentation for further -# detail and examples - -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/apps/backend/alembic/env.py b/apps/backend/alembic/env.py deleted file mode 100644 index 020b1b3..0000000 --- a/apps/backend/alembic/env.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Alembic environment configuration. -""" -from logging.config import fileConfig -from sqlalchemy import engine_from_config -from sqlalchemy import pool -from alembic import context -import os -import sys -from pathlib import Path - -# Add project root to path -sys.path.append(str(Path(__file__).parent.parent)) - -from app.models.database import Base -from app.config import get_settings - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Get settings -settings = get_settings() - -# Override database URL from environment -config.set_main_option('sqlalchemy.url', str(settings.database_url)) - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -target_metadata = Base.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - """ - connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/apps/backend/alembic/script.py.mako b/apps/backend/alembic/script.py.mako deleted file mode 100644 index 55df286..0000000 --- a/apps/backend/alembic/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - - -def upgrade() -> None: - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - ${downgrades if downgrades else "pass"} diff --git a/apps/backend/alembic/versions/001_initial_schema.py b/apps/backend/alembic/versions/001_initial_schema.py deleted file mode 100644 index ef75e86..0000000 --- a/apps/backend/alembic/versions/001_initial_schema.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Initial database schema - -Revision ID: 001 -Revises: -Create Date: 2024-01-01 00:00:00.000000 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = '001_initial_schema' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade() -> None: - """Create initial database schema.""" - # Create api_keys table - op.create_table('api_keys', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('key', sa.String(length=64), nullable=False), - sa.Column('name', sa.String(length=255), nullable=False), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('last_used_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('is_active', sa.Boolean(), nullable=True), - sa.Column('rate_limit_override', sa.String(length=50), nullable=True), - sa.Column('metadata', sa.JSON(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_api_keys_active', 'api_keys', ['is_active']) - op.create_index(op.f('ix_api_keys_key'), 'api_keys', ['key'], unique=True) - - # Create search_requests table - op.create_table('search_requests', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('request_id', sa.String(length=36), nullable=False), - sa.Column('api_key_id', sa.Integer(), nullable=True), - sa.Column('query', sa.Text(), nullable=False), - sa.Column('engines', sa.JSON(), nullable=False), - sa.Column('max_results', sa.Integer(), nullable=False), - sa.Column('language', sa.String(length=2), nullable=True), - sa.Column('safe_search', sa.String(length=10), nullable=True), - sa.Column('search_time_ms', sa.Integer(), nullable=True), - sa.Column('scraping_time_ms', sa.Integer(), nullable=True), - sa.Column('total_time_ms', sa.Integer(), nullable=True), - sa.Column('results_count', sa.Integer(), nullable=True), - sa.Column('scraped_count', sa.Integer(), nullable=True), - sa.Column('cache_hit', sa.Boolean(), nullable=True), - sa.Column('cache_key', sa.String(length=64), nullable=True), - sa.Column('client_ip', sa.String(length=45), nullable=True), - sa.Column('user_agent', sa.Text(), nullable=True), - sa.Column('request_headers', sa.JSON(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(['api_key_id'], ['api_keys.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_search_requests_cache_key', 'search_requests', ['cache_key']) - op.create_index('idx_search_requests_created', 'search_requests', ['created_at']) - op.create_index('idx_search_requests_query', 'search_requests', ['query']) - op.create_index(op.f('ix_search_requests_request_id'), 'search_requests', ['request_id'], unique=True) - - # Create search_results table - op.create_table('search_results', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('request_id', sa.Integer(), nullable=False), - sa.Column('rank', sa.Integer(), nullable=False), - sa.Column('title', sa.Text(), nullable=False), - sa.Column('url', sa.Text(), nullable=False), - sa.Column('snippet', sa.Text(), nullable=True), - sa.Column('engine', sa.String(length=50), nullable=False), - sa.Column('score', sa.Float(), nullable=True), - sa.Column('scraped_successfully', sa.Boolean(), nullable=True), - sa.Column('scraped_content', sa.JSON(), nullable=True), - sa.Column('scraping_error', sa.Text(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.ForeignKeyConstraint(['request_id'], ['search_requests.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_search_results_request', 'search_results', ['request_id']) - op.create_index('idx_search_results_url', 'search_results', ['url']) - - # Create scraping_jobs table - op.create_table('scraping_jobs', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('job_id', sa.String(length=36), nullable=False), - sa.Column('task_id', sa.String(length=255), nullable=True), - sa.Column('urls', sa.JSON(), nullable=False), - sa.Column('config', sa.JSON(), nullable=False), - sa.Column('status', sa.String(length=20), nullable=False), - sa.Column('results', sa.JSON(), nullable=True), - sa.Column('error_message', sa.Text(), nullable=True), - sa.Column('webhook_url', sa.Text(), nullable=True), - sa.Column('webhook_attempts', sa.Integer(), nullable=True), - sa.Column('webhook_last_attempt', sa.DateTime(timezone=True), nullable=True), - sa.Column('webhook_success', sa.Boolean(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_scraping_jobs_created', 'scraping_jobs', ['created_at']) - op.create_index('idx_scraping_jobs_status', 'scraping_jobs', ['status']) - op.create_index(op.f('ix_scraping_jobs_job_id'), 'scraping_jobs', ['job_id'], unique=True) - - # Create cache_entries table - op.create_table('cache_entries', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('cache_key', sa.String(length=64), nullable=False), - sa.Column('query_hash', sa.String(length=64), nullable=False), - sa.Column('size_bytes', sa.Integer(), nullable=True), - sa.Column('hit_count', sa.Integer(), nullable=True), - sa.Column('ttl_seconds', sa.Integer(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('last_accessed_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_cache_entries_expires', 'cache_entries', ['expires_at']) - op.create_index('idx_cache_entries_key', 'cache_entries', ['cache_key']) - op.create_index(op.f('ix_cache_entries_cache_key'), 'cache_entries', ['cache_key'], unique=True) - - # Create error_logs table - op.create_table('error_logs', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('request_id', sa.String(length=36), nullable=True), - sa.Column('error_type', sa.String(length=100), nullable=False), - sa.Column('error_message', sa.Text(), nullable=False), - sa.Column('error_details', sa.JSON(), nullable=True), - sa.Column('stack_trace', sa.Text(), nullable=True), - sa.Column('endpoint', sa.String(length=255), nullable=True), - sa.Column('method', sa.String(length=10), nullable=True), - sa.Column('status_code', sa.Integer(), nullable=True), - sa.Column('client_ip', sa.String(length=45), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_error_logs_created', 'error_logs', ['created_at']) - op.create_index('idx_error_logs_request', 'error_logs', ['request_id']) - op.create_index('idx_error_logs_type', 'error_logs', ['error_type']) - - -def downgrade() -> None: - """Drop all tables.""" - op.drop_index('idx_error_logs_type', table_name='error_logs') - op.drop_index('idx_error_logs_request', table_name='error_logs') - op.drop_index('idx_error_logs_created', table_name='error_logs') - op.drop_table('error_logs') - - op.drop_index(op.f('ix_cache_entries_cache_key'), table_name='cache_entries') - op.drop_index('idx_cache_entries_key', table_name='cache_entries') - op.drop_index('idx_cache_entries_expires', table_name='cache_entries') - op.drop_table('cache_entries') - - op.drop_index(op.f('ix_scraping_jobs_job_id'), table_name='scraping_jobs') - op.drop_index('idx_scraping_jobs_status', table_name='scraping_jobs') - op.drop_index('idx_scraping_jobs_created', table_name='scraping_jobs') - op.drop_table('scraping_jobs') - - op.drop_index('idx_search_results_url', table_name='search_results') - op.drop_index('idx_search_results_request', table_name='search_results') - op.drop_table('search_results') - - op.drop_index(op.f('ix_search_requests_request_id'), table_name='search_requests') - op.drop_index('idx_search_requests_query', table_name='search_requests') - op.drop_index('idx_search_requests_created', table_name='search_requests') - op.drop_index('idx_search_requests_cache_key', table_name='search_requests') - op.drop_table('search_requests') - - op.drop_index(op.f('ix_api_keys_key'), table_name='api_keys') - op.drop_index('idx_api_keys_active', table_name='api_keys') - op.drop_table('api_keys') diff --git a/apps/backend/alembic/versions/002_add_user_billing_tables.py b/apps/backend/alembic/versions/002_add_user_billing_tables.py deleted file mode 100644 index ff8a5b0..0000000 --- a/apps/backend/alembic/versions/002_add_user_billing_tables.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Add user and billing tables - -Revision ID: 002_add_user_billing -Revises: 001_initial_schema -Create Date: 2024-01-14 10:00:00.000000 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = '002_add_user_billing' -down_revision = '001_initial_schema' -branch_labels = None -depends_on = None - - -def upgrade(): - # Create enum types if not exists (idempotent for re-runs) - op.execute(""" - DO $$ - BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'plantype') THEN - CREATE TYPE plantype AS ENUM ('FREE', 'PRO', 'ENTERPRISE'); - END IF; - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'subscriptionstatus') THEN - CREATE TYPE subscriptionstatus AS ENUM ('ACTIVE', 'TRIALING', 'CANCELLED', 'PAST_DUE', 'UNPAID', 'INCOMPLETE'); - END IF; - END - $$; - """) - - # Create users table - op.create_table( - 'users', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('uuid', sa.String(36), nullable=False), - sa.Column('email', sa.String(255), nullable=False), - sa.Column('username', sa.String(100), nullable=True), - sa.Column('password_hash', sa.String(255), nullable=False), - sa.Column('salt', sa.String(32), nullable=False), - sa.Column('full_name', sa.String(255), nullable=True), - sa.Column('company', sa.String(255), nullable=True), - sa.Column('phone', sa.String(20), nullable=True), - sa.Column('timezone', sa.String(50), nullable=True, server_default='UTC'), - sa.Column('is_active', sa.Boolean(), nullable=True, server_default='true'), - sa.Column('is_verified', sa.Boolean(), nullable=True, server_default='false'), - sa.Column('is_admin', sa.Boolean(), nullable=True, server_default='false'), - sa.Column('email_verified_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('verification_token', sa.String(255), nullable=True), - sa.Column('reset_token', sa.String(255), nullable=True), - sa.Column('reset_token_expires', sa.DateTime(timezone=True), nullable=True), - sa.Column('stripe_customer_id', sa.String(255), nullable=True), - sa.Column('stripe_payment_method_id', sa.String(255), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('last_login_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_users_email', 'users', ['email'], unique=True) - op.create_index('idx_users_username', 'users', ['username'], unique=True) - op.create_index('idx_users_uuid', 'users', ['uuid'], unique=True) - op.create_index('idx_users_stripe', 'users', ['stripe_customer_id'], unique=True) - op.create_index('idx_users_active', 'users', ['is_active']) - - # Create user_api_keys table - op.create_table( - 'user_api_keys', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('key', sa.String(64), nullable=False), - sa.Column('name', sa.String(255), nullable=False), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('scopes', sa.JSON(), nullable=True), - sa.Column('ip_whitelist', sa.JSON(), nullable=True), - sa.Column('last_used_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('request_count', sa.Integer(), nullable=True, server_default='0'), - sa.Column('is_active', sa.Boolean(), nullable=True, server_default='true'), - sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_user_api_keys_key', 'user_api_keys', ['key'], unique=True) - op.create_index('idx_user_api_keys_active', 'user_api_keys', ['is_active']) - op.create_index('idx_user_api_keys_user', 'user_api_keys', ['user_id']) - - # Create plans table - op.create_table( - 'plans', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(100), nullable=False), - sa.Column('display_name', sa.String(255), nullable=False), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('stripe_product_id', sa.String(255), nullable=True), - sa.Column('stripe_price_id', sa.String(255), nullable=True), - sa.Column('price', sa.Float(), nullable=False), - sa.Column('currency', sa.String(3), nullable=True, server_default='usd'), - sa.Column('interval', sa.String(20), nullable=True, server_default='month'), - sa.Column('search_limit', sa.Integer(), nullable=True), - sa.Column('scrape_limit', sa.Integer(), nullable=True), - sa.Column('rate_limit', sa.String(50), nullable=True), - sa.Column('concurrent_requests', sa.Integer(), nullable=True, server_default='10'), - sa.Column('features', sa.JSON(), nullable=True), - sa.Column('is_active', sa.Boolean(), nullable=True, server_default='true'), - sa.Column('is_visible', sa.Boolean(), nullable=True, server_default='true'), - sa.Column('metadata', sa.JSON(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_plans_name', 'plans', ['name'], unique=True) - op.create_index('idx_plans_stripe_product', 'plans', ['stripe_product_id'], unique=True) - op.create_index('idx_plans_stripe_price', 'plans', ['stripe_price_id'], unique=True) - op.create_index('idx_plans_active', 'plans', ['is_active']) - - # Create subscriptions table - op.create_table( - 'subscriptions', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('stripe_subscription_id', sa.String(255), nullable=True), - sa.Column('stripe_price_id', sa.String(255), nullable=True), - sa.Column('stripe_product_id', sa.String(255), nullable=True), - sa.Column('plan_type', postgresql.ENUM('FREE', 'PRO', 'ENTERPRISE', name='plantype', create_type=False), nullable=False), - sa.Column('status', postgresql.ENUM('ACTIVE', 'TRIALING', 'CANCELLED', 'PAST_DUE', 'UNPAID', 'INCOMPLETE', name='subscriptionstatus', create_type=False), nullable=False), - sa.Column('amount', sa.Float(), nullable=True, server_default='0'), - sa.Column('currency', sa.String(3), nullable=True, server_default='usd'), - sa.Column('interval', sa.String(20), nullable=True, server_default='month'), - sa.Column('search_limit', sa.Integer(), nullable=True, server_default='1000'), - sa.Column('scrape_limit', sa.Integer(), nullable=True, server_default='10000'), - sa.Column('rate_limit', sa.String(50), nullable=True, server_default='100/hour'), - sa.Column('features', sa.JSON(), nullable=True), - sa.Column('trial_start', sa.DateTime(timezone=True), nullable=True), - sa.Column('trial_end', sa.DateTime(timezone=True), nullable=True), - sa.Column('current_period_start', sa.DateTime(timezone=True), nullable=True), - sa.Column('current_period_end', sa.DateTime(timezone=True), nullable=True), - sa.Column('cancelled_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('ended_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_subscriptions_user', 'subscriptions', ['user_id']) - op.create_index('idx_subscriptions_status', 'subscriptions', ['status']) - op.create_index('idx_subscriptions_stripe', 'subscriptions', ['stripe_subscription_id'], unique=True) - - # Create usage_records table - op.create_table( - 'usage_records', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('period_start', sa.DateTime(timezone=True), nullable=False), - sa.Column('period_end', sa.DateTime(timezone=True), nullable=False), - sa.Column('search_count', sa.Integer(), nullable=True, server_default='0'), - sa.Column('scrape_count', sa.Integer(), nullable=True, server_default='0'), - sa.Column('api_calls', sa.Integer(), nullable=True, server_default='0'), - sa.Column('usage_by_engine', sa.JSON(), nullable=True), - sa.Column('usage_by_day', sa.JSON(), nullable=True), - sa.Column('search_overage', sa.Integer(), nullable=True, server_default='0'), - sa.Column('scrape_overage', sa.Integer(), nullable=True, server_default='0'), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_usage_records_user', 'usage_records', ['user_id']) - op.create_index('idx_usage_records_period', 'usage_records', ['period_start', 'period_end']) - op.create_index('idx_usage_user_period', 'usage_records', ['user_id', 'period_start', 'period_end'], unique=True) - - # Create invoices table - op.create_table( - 'invoices', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column('stripe_invoice_id', sa.String(255), nullable=True), - sa.Column('stripe_charge_id', sa.String(255), nullable=True), - sa.Column('invoice_number', sa.String(100), nullable=True), - sa.Column('status', sa.String(50), nullable=True), - sa.Column('amount_due', sa.Integer(), nullable=True), - sa.Column('amount_paid', sa.Integer(), nullable=True), - sa.Column('amount_remaining', sa.Integer(), nullable=True), - sa.Column('subtotal', sa.Integer(), nullable=True), - sa.Column('tax', sa.Integer(), nullable=True), - sa.Column('total', sa.Integer(), nullable=True), - sa.Column('currency', sa.String(3), nullable=True, server_default='usd'), - sa.Column('period_start', sa.DateTime(timezone=True), nullable=True), - sa.Column('period_end', sa.DateTime(timezone=True), nullable=True), - sa.Column('due_date', sa.DateTime(timezone=True), nullable=True), - sa.Column('paid_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('invoice_pdf', sa.String(500), nullable=True), - sa.Column('hosted_invoice_url', sa.String(500), nullable=True), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('metadata', sa.JSON(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_invoices_user', 'invoices', ['user_id']) - op.create_index('idx_invoices_stripe', 'invoices', ['stripe_invoice_id'], unique=True) - op.create_index('idx_invoices_number', 'invoices', ['invoice_number'], unique=True) - op.create_index('idx_invoices_status', 'invoices', ['status']) - - # Create webhook_events table - op.create_table( - 'webhook_events', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('stripe_event_id', sa.String(255), nullable=False), - sa.Column('event_type', sa.String(100), nullable=False), - sa.Column('processed', sa.Boolean(), nullable=True, server_default='false'), - sa.Column('processed_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('error_message', sa.Text(), nullable=True), - sa.Column('retry_count', sa.Integer(), nullable=True, server_default='0'), - sa.Column('data', sa.JSON(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index('idx_webhook_events_stripe_id', 'webhook_events', ['stripe_event_id'], unique=True) - op.create_index('idx_webhook_events_type', 'webhook_events', ['event_type']) - op.create_index('idx_webhook_events_processed', 'webhook_events', ['processed']) - - # Add user_id foreign key to existing api_keys table (if it exists) - with op.batch_alter_table('api_keys') as batch_op: - batch_op.add_column(sa.Column('user_id', sa.Integer(), nullable=True)) - batch_op.create_foreign_key('fk_api_keys_user_id', 'users', ['user_id'], ['id']) - batch_op.create_index('idx_api_keys_user_id', ['user_id']) - - # Insert default plans - op.execute(""" - INSERT INTO plans (name, display_name, description, price, search_limit, scrape_limit, rate_limit, features) - VALUES - ('free', 'Free Plan', 'Get started with basic features', 0, 1000, 10000, '100/hour', - '{"api_access": true, "webhook_support": false, "priority_support": false}'::jsonb), - ('pro', 'Pro Plan', 'Unlimited searches and scrapes', 20, NULL, NULL, '1000/hour', - '{"api_access": true, "webhook_support": true, "priority_support": true, "custom_engines": true}'::jsonb), - ('enterprise', 'Enterprise Plan', 'Custom limits and dedicated support', 100, NULL, NULL, '10000/hour', - '{"api_access": true, "webhook_support": true, "priority_support": true, "custom_engines": true, "dedicated_pool": true, "sla": true}'::jsonb) - """) - - -def downgrade(): - # Drop tables in reverse order - op.drop_index('idx_api_keys_user_id', 'api_keys') - with op.batch_alter_table('api_keys') as batch_op: - batch_op.drop_constraint('fk_api_keys_user_id', type_='foreignkey') - batch_op.drop_column('user_id') - - op.drop_table('webhook_events') - op.drop_table('invoices') - op.drop_table('usage_records') - op.drop_table('subscriptions') - op.drop_table('plans') - op.drop_table('user_api_keys') - op.drop_table('users') - - # Drop enum types - op.execute("DROP TYPE IF EXISTS plantype") - op.execute("DROP TYPE IF EXISTS subscriptionstatus") diff --git a/apps/backend/app/__init__.py b/apps/backend/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/backend/app/api/__init__.py b/apps/backend/app/api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/backend/app/api/dependencies.py b/apps/backend/app/api/dependencies.py deleted file mode 100644 index 04aed27..0000000 --- a/apps/backend/app/api/dependencies.py +++ /dev/null @@ -1,118 +0,0 @@ -""" -FastAPI dependencies for dependency injection. -""" -from typing import Optional, Annotated -from fastapi import Depends, HTTPException, Header, Request, status -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -from sqlalchemy.ext.asyncio import AsyncSession - -from app.services.searxng import get_searxng_service, SearXNGService -from app.services.scraping import get_scraping_service, ContentScrapingService -from app.services.cache import get_cache_service, CacheService -from app.services.database import get_database_service, DatabaseService -from app.config import get_settings, Settings -import structlog - -logger = structlog.get_logger(__name__) - -# Security scheme -security = HTTPBearer(auto_error=False) - - -async def get_settings_dependency() -> Settings: - """Get application settings.""" - return get_settings() - - -async def get_db_service() -> DatabaseService: - """Get database service instance.""" - return await get_database_service() - - -async def get_searxng() -> SearXNGService: - """Get SearXNG service instance.""" - return await get_searxng_service() - - -async def get_scraper() -> ContentScrapingService: - """Get scraping service instance.""" - return await get_scraping_service() - - -async def get_cache() -> CacheService: - """Get cache service instance.""" - return await get_cache_service() - - -async def verify_api_key( - request: Request, - x_api_key: Optional[str] = Header(None, alias="X-API-Key"), - authorization: Optional[HTTPAuthorizationCredentials] = Depends(security), - settings: Settings = Depends(get_settings_dependency), - db: DatabaseService = Depends(get_db_service) -) -> Optional[str]: - """ - Verify API key from header or Bearer token. - - Returns: - API key ID if valid, None if no auth required - """ - # Check if API keys are configured - if not settings.api_keys: - # No API keys configured, allow access - return None - - # Try X-API-Key header first - api_key = x_api_key - - # Try Bearer token if no X-API-Key - if not api_key and authorization: - api_key = authorization.credentials - - if not api_key: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="API key required", - headers={"WWW-Authenticate": "Bearer"}, - ) - - # Validate API key - api_key_obj = await db.get_api_key(api_key) - - if not api_key_obj: - logger.warning( - "invalid_api_key", - api_key=api_key[:8] + "...", # Log partial key - client_ip=request.client.host - ) - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid API key", - headers={"WWW-Authenticate": "Bearer"}, - ) - - # Store API key info in request state for logging - request.state.api_key_id = api_key_obj.id - request.state.api_key_name = api_key_obj.name - - return api_key_obj.id - - -async def get_client_info(request: Request) -> dict: - """Extract client information from request.""" - return { - "client_ip": request.client.host if request.client else None, - "user_agent": request.headers.get("User-Agent"), - "referer": request.headers.get("Referer"), - "origin": request.headers.get("Origin") - } - - -# Type aliases for cleaner dependency injection -ApiKeyDep = Annotated[Optional[str], Depends(verify_api_key)] -SettingsDep = Annotated[Settings, Depends(get_settings_dependency)] -DatabaseDep = Annotated[DatabaseService, Depends(get_db_service)] -SearxngDep = Annotated[SearXNGService, Depends(get_searxng)] -ScraperDep = Annotated[ContentScrapingService, Depends(get_scraper)] -CacheDep = Annotated[CacheService, Depends(get_cache)] -ClientInfoDep = Annotated[dict, Depends(get_client_info)] diff --git a/apps/backend/app/api/v1/__init__.py b/apps/backend/app/api/v1/__init__.py deleted file mode 100644 index 9a43c21..0000000 --- a/apps/backend/app/api/v1/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -API v1 routes. -""" diff --git a/apps/backend/app/api/v1/auth.py b/apps/backend/app/api/v1/auth.py deleted file mode 100644 index 3f66090..0000000 --- a/apps/backend/app/api/v1/auth.py +++ /dev/null @@ -1,367 +0,0 @@ -""" -Authentication and user management endpoints. -""" -from typing import Optional -from fastapi import APIRouter, HTTPException, Depends, status, BackgroundTasks -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -import structlog - -from app.models.auth_models import ( - UserRegisterRequest, - UserLoginRequest, - RefreshTokenRequest, - CreateAPIKeyRequest, - ResetPasswordRequest, - ChangePasswordRequest, - UserResponse, - LoginResponse, - APIKeyResponse, - UsageResponse, - SubscriptionResponse, - OAuthSyncRequest, -) -from app.services.auth_service import get_auth_service, AuthService -from app.services.stripe_service import get_stripe_service, StripeService -from app.services.database import get_database_service, DatabaseService -from app.models.users import User -from app.utils.exceptions import UnauthorizedException, BadRequestException - -logger = structlog.get_logger(__name__) - -router = APIRouter(prefix="/auth", tags=["authentication"]) -security = HTTPBearer() - - -async def get_current_user( - credentials: HTTPAuthorizationCredentials = Depends(security), - auth_service: AuthService = Depends(get_auth_service), - db_service: DatabaseService = Depends(get_database_service) -) -> User: - """Get current authenticated user from JWT token.""" - token = credentials.credentials - - # First try as JWT token - user = await auth_service.verify_token(token) - - # If not JWT, try as API key - if not user: - user = await auth_service.verify_api_key(token) - - if not user: - raise UnauthorizedException("Invalid or expired token") - - return user - - -@router.post("/register", response_model=UserResponse, status_code=status.HTTP_201_CREATED) -async def register( - request: UserRegisterRequest, - background_tasks: BackgroundTasks, - auth_service: AuthService = Depends(get_auth_service), - stripe_service: StripeService = Depends(get_stripe_service) -): - """Register a new user account.""" - try: - # Create user - user = await auth_service.register_user( - email=request.email, - password=request.password, - full_name=request.full_name, - company=request.company - ) - - # Create Stripe customer - background_tasks.add_task(stripe_service.create_customer, user) - - # TODO: Send verification email - # background_tasks.add_task(send_verification_email, user) - - return UserResponse( - id=user.id, - email=user.email, - full_name=user.full_name, - company=user.company, - is_verified=user.is_verified, - plan="free", - created_at=user.created_at - ) - - except Exception as e: - logger.error("registration_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(e) - ) - - -@router.post("/login", response_model=LoginResponse) -async def login( - request: UserLoginRequest, - auth_service: AuthService = Depends(get_auth_service) -): - """Login with email and password.""" - try: - result = await auth_service.login(request.email, request.password) - return LoginResponse(**result) - - except UnauthorizedException as e: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail=str(e) - ) - except Exception as e: - logger.error("login_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Login failed" - ) - - -@router.post("/refresh", response_model=LoginResponse) -async def refresh_token( - request: RefreshTokenRequest, - auth_service: AuthService = Depends(get_auth_service) -): - """Refresh access token using refresh token.""" - try: - result = await auth_service.refresh_tokens(request.refresh_token) - return LoginResponse(**result) - - except UnauthorizedException as e: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail=str(e) - ) - - -@router.get("/me", response_model=UserResponse) -async def get_current_user_info( - user: User = Depends(get_current_user) -): - """Get current user information.""" - return UserResponse( - id=user.id, - email=user.email, - full_name=user.full_name, - company=user.company, - is_verified=user.is_verified, - plan=user.current_plan.value if user.current_plan else "free", - created_at=user.created_at - ) - - -@router.put("/me", response_model=UserResponse) -async def update_user( - full_name: Optional[str] = None, - company: Optional[str] = None, - timezone: Optional[str] = None, - user: User = Depends(get_current_user), - db_service: DatabaseService = Depends(get_database_service) -): - """Update current user information.""" - if full_name: - user.full_name = full_name - if company: - user.company = company - if timezone: - user.timezone = timezone - - await db_service.update_user(user) - - return UserResponse( - id=user.id, - email=user.email, - full_name=user.full_name, - company=user.company, - is_verified=user.is_verified, - plan=user.current_plan.value if user.current_plan else "free", - created_at=user.created_at - ) - - -@router.post("/api-keys", response_model=APIKeyResponse) -async def create_api_key( - request: CreateAPIKeyRequest, - user: User = Depends(get_current_user), - auth_service: AuthService = Depends(get_auth_service) -): - """Create a new API key.""" - api_key = await auth_service.create_api_key( - user=user, - name=request.name, - description=request.description, - scopes=request.scopes - ) - - return APIKeyResponse( - id=api_key.id, - key=api_key.key, # Only shown once - name=api_key.name, - description=api_key.description, - scopes=api_key.scopes, - created_at=api_key.created_at - ) - - -@router.get("/api-keys", response_model=list[APIKeyResponse]) -async def list_api_keys( - user: User = Depends(get_current_user), - db_service: DatabaseService = Depends(get_database_service) -): - """List all API keys for current user.""" - api_keys = await db_service.get_user_api_keys(user.id) - - return [ - APIKeyResponse( - id=key.id, - key="sk_****" + key.key[-8:], # Partially hidden - name=key.name, - description=key.description, - scopes=key.scopes, - last_used_at=key.last_used_at, - created_at=key.created_at - ) - for key in api_keys - ] - - -@router.delete("/api-keys/{key_id}") -async def delete_api_key( - key_id: int, - user: User = Depends(get_current_user), - db_service: DatabaseService = Depends(get_database_service) -): - """Delete an API key.""" - success = await db_service.delete_api_key(key_id, user.id) - - if not success: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="API key not found" - ) - - return {"message": "API key deleted"} - - -@router.get("/usage", response_model=UsageResponse) -async def get_usage( - user: User = Depends(get_current_user), - auth_service: AuthService = Depends(get_auth_service) -): - """Get current usage statistics.""" - usage = await auth_service.get_user_usage(user) - return UsageResponse(**usage) - - -@router.post("/verify-email") -async def verify_email( - token: str, - auth_service: AuthService = Depends(get_auth_service) -): - """Verify email address.""" - try: - success = await auth_service.verify_email(token) - return {"message": "Email verified successfully"} - except Exception as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(e) - ) - - -@router.post("/reset-password") -async def request_password_reset( - request: ResetPasswordRequest, - background_tasks: BackgroundTasks, - auth_service: AuthService = Depends(get_auth_service) -): - """Request password reset.""" - reset_token = await auth_service.reset_password_request(request.email) - - # TODO: Send reset email - # background_tasks.add_task(send_reset_email, request.email, reset_token) - - return {"message": "If the email exists, a reset link has been sent"} - - -@router.post("/reset-password/confirm") -async def reset_password( - token: str, - new_password: str, - auth_service: AuthService = Depends(get_auth_service) -): - """Reset password using token.""" - try: - success = await auth_service.reset_password(token, new_password) - return {"message": "Password reset successfully"} - except Exception as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(e) - ) - - -@router.post("/change-password") -async def change_password( - request: ChangePasswordRequest, - user: User = Depends(get_current_user), - auth_service: AuthService = Depends(get_auth_service), - db_service: DatabaseService = Depends(get_database_service) -): - """Change password for authenticated user.""" - # Verify current password - try: - await auth_service.login(user.email, request.current_password) - except: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Current password is incorrect" - ) - - # Change password - await auth_service.reset_password(user.reset_token, request.new_password) - - return {"message": "Password changed successfully"} - - -@router.post("/oauth-sync", response_model=LoginResponse) -async def oauth_sync( - request: OAuthSyncRequest, - auth_service: AuthService = Depends(get_auth_service), - db_service: DatabaseService = Depends(get_database_service) -): - """Upsert user from OAuth provider and return backend tokens. - - Flow: - - If user exists by email → return tokens - - Else create user (verified), set profile fields → return tokens - """ - # Try to find existing user - user = await db_service.get_user_by_email(request.email) - if not user: - # Create a random password for OAuth users (not used for login) - # Reuse AuthService to create user properly - user = await auth_service.register_user( - email=request.email, - password=jwt.encode({"rand": request.oauth_id}, settings.secret_key, algorithm=settings.jwt_algorithm), - full_name=request.full_name - ) - user.is_verified = True - await db_service.update_user(user) - - # Issue tokens - access_token = auth_service.create_access_token(user) - refresh_token = auth_service.create_refresh_token(user) - return LoginResponse( - access_token=access_token, - refresh_token=refresh_token, - token_type="bearer", - expires_in=auth_service.access_token_expire_minutes * 60, - user={ - "id": user.id, - "email": user.email, - "full_name": user.full_name, - "is_verified": user.is_verified, - "plan": user.current_plan.value if user.current_plan else "free", - }, - ) diff --git a/apps/backend/app/api/v1/billing.py b/apps/backend/app/api/v1/billing.py deleted file mode 100644 index e82fc36..0000000 --- a/apps/backend/app/api/v1/billing.py +++ /dev/null @@ -1,413 +0,0 @@ -""" -Billing and subscription management endpoints. -""" -from typing import Optional, List -from fastapi import APIRouter, HTTPException, Depends, Request, status, Header -from fastapi.responses import RedirectResponse -import structlog -import stripe - -from app.models.auth_models import ( - CreateCheckoutSessionRequest, - CreateSubscriptionRequest, - UpdateSubscriptionRequest, - SubscriptionResponse, - PlanResponse, - InvoiceResponse, - CheckoutSessionResponse, - BillingPortalResponse, -) -from app.services.auth_service import get_auth_service, AuthService -from app.services.stripe_service import get_stripe_service, StripeService -from app.services.database import get_database_service, DatabaseService -from app.models.users import User, Subscription, Plan, Invoice -from app.api.v1.auth import get_current_user -from app.config import get_settings - -logger = structlog.get_logger(__name__) -settings = get_settings() - -router = APIRouter(prefix="/billing", tags=["billing"]) - - -@router.get("/plans", response_model=List[PlanResponse]) -async def list_plans( - db_service: DatabaseService = Depends(get_database_service) -): - """List all available subscription plans.""" - plans = await db_service.get_active_plans() - - return [ - PlanResponse( - id=plan.id, - name=plan.name, - display_name=plan.display_name, - description=plan.description, - price=plan.price, - currency=plan.currency, - interval=plan.interval, - search_limit=plan.search_limit, - scrape_limit=plan.scrape_limit, - rate_limit=plan.rate_limit, - features=plan.features - ) - for plan in plans - ] - - -@router.get("/subscription", response_model=Optional[SubscriptionResponse]) -async def get_subscription( - user: User = Depends(get_current_user), - db_service: DatabaseService = Depends(get_database_service) -): - """Get current user subscription.""" - subscription = user.current_subscription - - if not subscription: - return None - - return SubscriptionResponse( - id=subscription.id, - plan_type=subscription.plan_type.value, - status=subscription.status.value, - amount=subscription.amount, - currency=subscription.currency, - interval=subscription.interval, - search_limit=subscription.search_limit, - scrape_limit=subscription.scrape_limit, - rate_limit=subscription.rate_limit, - features=subscription.features, - current_period_start=subscription.current_period_start, - current_period_end=subscription.current_period_end, - trial_end=subscription.trial_end, - cancelled_at=subscription.cancelled_at, - is_active=subscription.is_active, - days_remaining=subscription.days_remaining - ) - - -@router.post("/subscription", response_model=SubscriptionResponse) -async def create_subscription( - request: CreateSubscriptionRequest, - user: User = Depends(get_current_user), - stripe_service: StripeService = Depends(get_stripe_service) -): - """Create a new subscription.""" - # Check if user already has an active subscription - if user.current_subscription and user.current_subscription.is_active: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="User already has an active subscription" - ) - - try: - subscription = await stripe_service.create_subscription( - user=user, - price_id=request.price_id, - trial_days=request.trial_days or 0 - ) - - return SubscriptionResponse( - id=subscription.id, - plan_type=subscription.plan_type.value, - status=subscription.status.value, - amount=subscription.amount, - currency=subscription.currency, - interval=subscription.interval, - search_limit=subscription.search_limit, - scrape_limit=subscription.scrape_limit, - rate_limit=subscription.rate_limit, - features=subscription.features, - current_period_start=subscription.current_period_start, - current_period_end=subscription.current_period_end, - trial_end=subscription.trial_end, - is_active=subscription.is_active - ) - - except stripe.error.StripeError as e: - logger.error("subscription_creation_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Subscription creation failed: {str(e)}" - ) - - -@router.put("/subscription", response_model=SubscriptionResponse) -async def update_subscription( - request: UpdateSubscriptionRequest, - user: User = Depends(get_current_user), - stripe_service: StripeService = Depends(get_stripe_service) -): - """Update existing subscription to a different plan.""" - subscription = user.current_subscription - - if not subscription or not subscription.is_active: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="No active subscription found" - ) - - try: - subscription = await stripe_service.update_subscription( - subscription=subscription, - new_price_id=request.price_id - ) - - return SubscriptionResponse( - id=subscription.id, - plan_type=subscription.plan_type.value, - status=subscription.status.value, - amount=subscription.amount, - currency=subscription.currency, - interval=subscription.interval, - search_limit=subscription.search_limit, - scrape_limit=subscription.scrape_limit, - rate_limit=subscription.rate_limit, - features=subscription.features, - current_period_start=subscription.current_period_start, - current_period_end=subscription.current_period_end, - is_active=subscription.is_active - ) - - except stripe.error.StripeError as e: - logger.error("subscription_update_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Subscription update failed: {str(e)}" - ) - - -@router.delete("/subscription") -async def cancel_subscription( - immediately: bool = False, - user: User = Depends(get_current_user), - stripe_service: StripeService = Depends(get_stripe_service) -): - """Cancel current subscription.""" - subscription = user.current_subscription - - if not subscription or not subscription.is_active: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="No active subscription found" - ) - - try: - await stripe_service.cancel_subscription( - subscription=subscription, - immediately=immediately - ) - - return { - "message": f"Subscription {'cancelled immediately' if immediately else 'will be cancelled at period end'}" - } - - except stripe.error.StripeError as e: - logger.error("subscription_cancellation_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Subscription cancellation failed: {str(e)}" - ) - - -@router.post("/checkout-session", response_model=CheckoutSessionResponse) -async def create_checkout_session( - request: CreateCheckoutSessionRequest, - user: User = Depends(get_current_user), - stripe_service: StripeService = Depends(get_stripe_service) -): - """Create a Stripe Checkout session for subscription.""" - try: - checkout_url = await stripe_service.create_checkout_session( - user=user, - price_id=request.price_id, - success_url=request.success_url, - cancel_url=request.cancel_url, - trial_days=request.trial_days or 0 - ) - - return CheckoutSessionResponse( - checkout_url=checkout_url - ) - - except stripe.error.StripeError as e: - logger.error("checkout_session_creation_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Checkout session creation failed: {str(e)}" - ) - - -@router.post("/billing-portal", response_model=BillingPortalResponse) -async def create_billing_portal_session( - return_url: str, - user: User = Depends(get_current_user), - stripe_service: StripeService = Depends(get_stripe_service) -): - """Create a Stripe Billing Portal session for subscription management.""" - try: - portal_url = await stripe_service.create_billing_portal_session( - user=user, - return_url=return_url - ) - - return BillingPortalResponse( - portal_url=portal_url - ) - - except Exception as e: - logger.error("billing_portal_session_creation_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Billing portal session creation failed: {str(e)}" - ) - - -@router.get("/invoices", response_model=List[InvoiceResponse]) -async def list_invoices( - limit: int = 10, - user: User = Depends(get_current_user), - db_service: DatabaseService = Depends(get_database_service) -): - """List user invoices.""" - invoices = await db_service.get_user_invoices(user.id, limit) - - return [ - InvoiceResponse( - id=invoice.id, - invoice_number=invoice.invoice_number, - status=invoice.status, - amount_due=invoice.amount_due / 100, # Convert from cents - amount_paid=invoice.amount_paid / 100 if invoice.amount_paid else 0, - currency=invoice.currency, - period_start=invoice.period_start, - period_end=invoice.period_end, - paid_at=invoice.paid_at, - invoice_pdf=invoice.invoice_pdf, - hosted_invoice_url=invoice.hosted_invoice_url, - created_at=invoice.created_at - ) - for invoice in invoices - ] - - -@router.post("/webhook/stripe") -async def stripe_webhook( - request: Request, - stripe_signature: str = Header(None, alias="Stripe-Signature"), - stripe_service: StripeService = Depends(get_stripe_service) -): - """Handle Stripe webhook events.""" - # Get the raw body - body = await request.body() - - # Handle the webhook - success = await stripe_service.handle_webhook( - payload=body.decode('utf-8'), - signature=stripe_signature - ) - - if not success: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Webhook processing failed" - ) - - return {"received": True} - - -@router.get("/payment-methods") -async def list_payment_methods( - user: User = Depends(get_current_user) -): - """List user payment methods.""" - if not user.stripe_customer_id: - return [] - - try: - payment_methods = stripe.PaymentMethod.list( - customer=user.stripe_customer_id, - type="card" - ) - - return [ - { - "id": pm.id, - "brand": pm.card.brand, - "last4": pm.card.last4, - "exp_month": pm.card.exp_month, - "exp_year": pm.card.exp_year, - "is_default": pm.id == user.stripe_payment_method_id - } - for pm in payment_methods.data - ] - - except stripe.error.StripeError as e: - logger.error("payment_methods_fetch_failed", error=str(e)) - return [] - - -@router.post("/payment-methods") -async def add_payment_method( - payment_method_id: str, - set_as_default: bool = True, - user: User = Depends(get_current_user), - db_service: DatabaseService = Depends(get_database_service) -): - """Add a payment method to user account.""" - if not user.stripe_customer_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="User has no Stripe customer account" - ) - - try: - # Attach payment method to customer - payment_method = stripe.PaymentMethod.attach( - payment_method_id, - customer=user.stripe_customer_id - ) - - # Set as default if requested - if set_as_default: - stripe.Customer.modify( - user.stripe_customer_id, - invoice_settings={"default_payment_method": payment_method_id} - ) - user.stripe_payment_method_id = payment_method_id - await db_service.update_user(user) - - return { - "message": "Payment method added successfully", - "payment_method": { - "id": payment_method.id, - "brand": payment_method.card.brand, - "last4": payment_method.card.last4 - } - } - - except stripe.error.StripeError as e: - logger.error("payment_method_add_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to add payment method: {str(e)}" - ) - - -@router.delete("/payment-methods/{payment_method_id}") -async def remove_payment_method( - payment_method_id: str, - user: User = Depends(get_current_user) -): - """Remove a payment method.""" - try: - stripe.PaymentMethod.detach(payment_method_id) - - return {"message": "Payment method removed successfully"} - - except stripe.error.StripeError as e: - logger.error("payment_method_removal_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Failed to remove payment method: {str(e)}" - ) diff --git a/apps/backend/app/api/v1/enhanced_search.py b/apps/backend/app/api/v1/enhanced_search.py deleted file mode 100644 index e286e41..0000000 --- a/apps/backend/app/api/v1/enhanced_search.py +++ /dev/null @@ -1,783 +0,0 @@ -""" -Enhanced search and scraping API endpoints with crawl4ai-inspired capabilities. - -This module provides advanced search and scraping functionality including: -- Advanced extraction strategies -- Content filtering -- Enhanced markdown generation -- Adaptive crawling -- Virtual scrolling -- Link analysis -""" - -import asyncio -import uuid -from typing import List, Dict, Any -from datetime import datetime -from fastapi import APIRouter, HTTPException, BackgroundTasks, Request, status -from fastapi.responses import JSONResponse, Response -import structlog - -from app.models.requests import ( - UnQuestRequest, BatchSearchRequest, ScrapingConfig, - ExtractionStrategyConfig, ContentFilterConfig, MarkdownConfig, - AdaptiveCrawlConfig, VirtualScrollConfig, LinkAnalysisConfig -) -from app.models.responses import ( - UnQuestResponse, AsyncTaskResponse, BatchSearchResponse, - SearchResult, SearchMetadata, EnginesListResponse, HealthResponse, ServiceHealth -) -from app.api.dependencies import ( - ApiKeyDep, SettingsDep, DatabaseDep, SearxngDep, - ScraperDep, CacheDep, ClientInfoDep -) -from app.workers.tasks import process_async_search_scrape -from app.services.enhanced_scraping import get_enhanced_scraping_service -from app.services.multi_search import get_multi_search_service, SearchOptions -from app.services.multi_engine_scraper import get_multi_engine_service -from app.services.llm_configuration import get_llm_config_service, generate_config_from_prompt -from app.services.batch_operations import get_batch_service -from app.services.multi_entity_extraction import get_multi_entity_service, MultiEntityExtractionRequest - -logger = structlog.get_logger(__name__) - -router = APIRouter(prefix="/enhanced", tags=["enhanced-search"]) - - -@router.post("/search", response_model=UnQuestResponse) -async def enhanced_search_and_scrape( - request_data: UnQuestRequest, - request: Request, - background_tasks: BackgroundTasks, - api_key_id: ApiKeyDep, - settings: SettingsDep, - db: DatabaseDep, - searxng: SearxngDep, - scraper: ScraperDep, - cache: CacheDep, - client_info: ClientInfoDep -): - """ - Enhanced search and scrape endpoint with crawl4ai-inspired capabilities. - - This endpoint provides advanced features including: - - Multiple extraction strategies (Cosine, JSON CSS, Regex, LLM) - - Content filtering (BM25, Pruning, LLM) - - Enhanced markdown generation with citations - - Adaptive crawling with learning algorithms - - Virtual scrolling for infinite pages - - Intelligent link analysis and scoring - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - # Generate cache key - cache_key = cache.generate_cache_key(request_data) - - # Check cache if enabled - if request_data.cache_ttl > 0: - cached_response = await cache.get_search_results(cache_key) - if cached_response: - cached_response.request_id = request_id - - # Log request with cached response - await db.log_search_request( - request_data.dict(), - cached_response, - api_key_id, - client_info["client_ip"], - client_info["user_agent"] - ) - - return cached_response - - # Handle async mode - if request_data.async_mode: - if not request_data.webhook_url: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="webhook_url is required for async mode" - ) - - # Create enhanced scraping job - job = await db.create_scraping_job( - urls=[], # Will be populated after search - config=request_data.dict(), - webhook_url=str(request_data.webhook_url) - ) - - # Queue async task with enhanced processing - task = process_async_search_scrape.delay( - job_id=job.job_id, - request_data=request_data.dict(), - enhanced_processing=True - ) - - # Update job with task ID - await db.update_scraping_job( - job.job_id, - status="processing", - task_id=task.id - ) - - return AsyncTaskResponse( - task_id=job.job_id, - status="processing", - message="Enhanced search and scraping job queued", - estimated_completion_time=datetime.utcnow().timestamp() + 60 - ) - - # Perform search - logger.info("enhanced_search_started", query=request_data.query, engines=request_data.engines) - - search_results = await searxng.search( - query=request_data.query, - engines=request_data.engines, - num_results=request_data.max_results, - language=request_data.language, - safe_search=request_data.safe_search, - timeout=request_data.timeout - ) - - if not search_results: - return UnQuestResponse( - request_id=request_id, - query=request_data.query, - results=[], - metadata=SearchMetadata( - total_results=0, - engines_used=request_data.engines, - search_time_ms=int((asyncio.get_event_loop().time() - start_time) * 1000), - language=request_data.language - ) - ) - - # Prepare enhanced scraping configuration - enhanced_scraping_config = await _prepare_enhanced_scraping_config(request_data) - - # Enhanced content scraping - scraped_contents = [] - if request_data.scrape_content: - logger.info("enhanced_content_scraping_started", urls=len(search_results)) - - # Get enhanced scraping service - enhanced_scraper = await get_enhanced_scraping_service() - - # Extract URLs from search results - urls_to_scrape = [result.url for result in search_results] - - # Perform enhanced scraping - scraped_contents = await enhanced_scraper.scrape_urls_enhanced( - urls=urls_to_scrape, - config=enhanced_scraping_config - ) - - # Process and combine results - results = await _combine_search_and_scraping_results( - search_results, scraped_contents, request_data - ) - - # Calculate processing time - processing_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) - - # Create response - response = UnQuestResponse( - request_id=request_id, - query=request_data.query, - results=results, - metadata=SearchMetadata( - total_results=len(results), - engines_used=request_data.engines, - search_time_ms=processing_time_ms, - language=request_data.language, - scraped_count=len(scraped_contents), - enhanced_features_used=_get_enhanced_features_summary(request_data) - ) - ) - - # Cache response if enabled - if request_data.cache_ttl > 0: - await cache.set_search_results(cache_key, response, request_data.cache_ttl) - - # Log successful request - await db.log_search_request( - request_data.dict(), - response, - api_key_id, - client_info["client_ip"], - client_info["user_agent"] - ) - - logger.info( - "enhanced_search_completed", - request_id=request_id, - results_count=len(results), - processing_time_ms=processing_time_ms - ) - - return response - - except Exception as e: - logger.error("enhanced_search_failed", request_id=request_id, error=str(e)) - - # Log failed request - await db.log_search_request( - request_data.dict(), - None, - api_key_id, - client_info["client_ip"], - client_info["user_agent"], - error=str(e) - ) - - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Enhanced search failed: {str(e)}" - ) - - -@router.post("/scrape", response_model=Dict[str, Any]) -async def enhanced_scraping_only( - config: ScrapingConfig, - api_key_id: ApiKeyDep, - settings: SettingsDep, - db: DatabaseDep, - client_info: ClientInfoDep -): - """ - Enhanced content scraping endpoint without search. - - Directly scrapes provided URLs with all advanced features. - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - logger.info( - "enhanced_scraping_only_started", - urls=len(config.urls), - extraction_strategy=getattr(config, 'extraction_strategy', 'none') - ) - - # Get enhanced scraping service - enhanced_scraper = await get_enhanced_scraping_service() - - # Convert URLs to strings - urls = [str(url) for url in config.urls] - - # Perform enhanced scraping - scraped_contents = await enhanced_scraper.scrape_urls_enhanced( - urls=urls, - config=config - ) - - processing_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) - - # Prepare response - response = { - "request_id": request_id, - "scraped_content": [ - { - "url": content.url, - "title": content.title, - "text": content.text, - "extraction_success": content.extraction_success, - "word_count": content.word_count, - "content_quality_score": content.content_quality_score, - **({ - "extracted_content": getattr(content, 'extracted_content', None) - } if hasattr(content, 'extracted_content') else {}), - **({ - "markdown": getattr(content, 'markdown', None) - } if hasattr(content, 'markdown') else {}), - **({ - "link_analysis": getattr(content, 'link_analysis', None) - } if hasattr(content, 'link_analysis') else {}) - } - for content in scraped_contents - ], - "metadata": { - "total_urls": len(urls), - "successful_scrapes": sum(1 for c in scraped_contents if c.extraction_success), - "processing_time_ms": processing_time_ms, - "enhanced_features": _get_enhanced_features_summary_from_config(config) - } - } - - # Log scraping request - await db.log_scraping_request( - config.dict(), - response, - api_key_id, - client_info["client_ip"], - client_info["user_agent"] - ) - - logger.info( - "enhanced_scraping_only_completed", - request_id=request_id, - successful_scrapes=response["metadata"]["successful_scrapes"], - processing_time_ms=processing_time_ms - ) - - return response - - except Exception as e: - logger.error("enhanced_scraping_only_failed", request_id=request_id, error=str(e)) - - # Log failed request - await db.log_scraping_request( - config.dict(), - None, - api_key_id, - client_info["client_ip"], - client_info["user_agent"], - error=str(e) - ) - - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Enhanced scraping failed: {str(e)}" - ) - - -@router.get("/features", response_model=Dict[str, Any]) -async def get_enhanced_features(): - """Get information about available enhanced features.""" - return { - "extraction_strategies": { - "none": "No extraction - return raw content", - "cosine": "Semantic similarity clustering for content extraction", - "json_css": "Structured data extraction using CSS selectors and schemas", - "regex": "Pattern-based extraction using regular expressions", - "llm": "AI-powered structured data extraction using language models" - }, - "content_filters": { - "none": "No filtering - return all content", - "pruning": "Remove irrelevant content based on configurable thresholds", - "bm25": "Information retrieval-based filtering using BM25 algorithm", - "llm": "AI-powered content relevance filtering" - }, - "markdown_generation": { - "features": [ - "Enhanced HTML to markdown conversion", - "Citation management and link analysis", - "Multiple output formats (raw, fit, with references)", - "Link prioritization and scoring" - ] - }, - "adaptive_crawling": { - "features": [ - "Learning algorithms that improve extraction over time", - "Statistical and embedding-based strategies", - "Information saturation detection", - "State persistence for continued learning" - ] - }, - "virtual_scrolling": { - "features": [ - "Automatic infinite scroll detection and handling", - "Smart waiting strategies for dynamic content", - "Content extraction during scrolling", - "Progress tracking and optimization" - ] - }, - "link_analysis": { - "features": [ - "3-layer scoring system for smart link prioritization", - "Domain authority and credibility assessment", - "Content relevance scoring", - "Link quality metrics and filtering" - ] - } - } - - -async def _prepare_enhanced_scraping_config(request_data: UnQuestRequest) -> ScrapingConfig: - """Prepare enhanced scraping configuration from request data.""" - config_dict = { - "urls": [], # Will be populated with search results - "selectors": request_data.scrape_selectors, - "extract_text": True, - "extract_images": request_data.include_images, - "extract_links": request_data.include_links, - "extract_metadata": True, - "javascript_rendering": request_data.js_mode, - "js_mode": request_data.js_mode, - "wait_time": 3 if request_data.js_mode else 0, - "response_format": request_data.output_format, - "screenshot": getattr(request_data, 'screenshot', False), - "pdf": getattr(request_data, 'pdf', False), - "include_html": False, - "cache_mode": "enabled", - "cache_ttl": request_data.cache_ttl, - "per_host_concurrency": 2, - "hits_per_sec": 1.0, - } - - # Add enhanced features if specified in request - enhanced_features = [ - 'extraction_strategy', 'extraction_config', - 'content_filter', 'content_filter_config', - 'markdown_generation', 'markdown_config', - 'adaptive_crawling', 'adaptive_config', - 'virtual_scrolling', 'virtual_scroll_config', - 'link_analysis', 'link_analysis_config' - ] - - for feature in enhanced_features: - if hasattr(request_data, feature): - config_dict[feature] = getattr(request_data, feature) - - return ScrapingConfig(**config_dict) - - -async def _combine_search_and_scraping_results( - search_results: List[Any], - scraped_contents: List[Any], - request_data: UnQuestRequest -) -> List[SearchResult]: - """Combine search results with scraped content.""" - combined_results = [] - - # Create lookup for scraped content - scraped_lookup = {content.url: content for content in scraped_contents} - - for search_result in search_results: - # Get corresponding scraped content - scraped_content = scraped_lookup.get(search_result.url) - - # Create enhanced search result - result_dict = { - "title": search_result.title, - "url": search_result.url, - "description": search_result.description, - "engine": search_result.engine, - "score": getattr(search_result, 'score', 0.0) - } - - # Add scraped content if available - if scraped_content and scraped_content.extraction_success: - result_dict.update({ - "content": scraped_content.text, - "word_count": scraped_content.word_count, - "language": scraped_content.language_detected, - "quality_score": scraped_content.content_quality_score, - "images": scraped_content.images if request_data.include_images else [], - "links": scraped_content.links if request_data.include_links else [], - "metadata": { - "author": scraped_content.metadata.author, - "published_date": scraped_content.metadata.published_date, - "keywords": scraped_content.metadata.keywords - } - }) - - # Add enhanced features if present - if hasattr(scraped_content, 'extracted_content'): - result_dict["extracted_content"] = scraped_content.extracted_content - - if hasattr(scraped_content, 'markdown'): - result_dict["markdown"] = scraped_content.markdown - - if hasattr(scraped_content, 'link_analysis'): - result_dict["link_analysis"] = scraped_content.link_analysis - - combined_results.append(SearchResult(**result_dict)) - - return combined_results - - -def _get_enhanced_features_summary(request_data: UnQuestRequest) -> Dict[str, Any]: - """Get summary of enhanced features used in request.""" - features = {} - - if hasattr(request_data, 'extraction_strategy') and request_data.extraction_strategy != 'none': - features['extraction_strategy'] = request_data.extraction_strategy - - if hasattr(request_data, 'content_filter') and request_data.content_filter != 'none': - features['content_filter'] = request_data.content_filter - - if hasattr(request_data, 'markdown_generation') and request_data.markdown_generation: - features['markdown_generation'] = True - - if hasattr(request_data, 'adaptive_crawling') and request_data.adaptive_crawling: - features['adaptive_crawling'] = True - - if hasattr(request_data, 'virtual_scrolling') and request_data.virtual_scrolling: - features['virtual_scrolling'] = True - - if hasattr(request_data, 'link_analysis') and request_data.link_analysis: - features['link_analysis'] = True - - return features - - -def _get_enhanced_features_summary_from_config(config: ScrapingConfig) -> Dict[str, Any]: - """Get summary of enhanced features from scraping config.""" - features = {} - - if hasattr(config, 'extraction_strategy') and config.extraction_strategy != 'none': - features['extraction_strategy'] = config.extraction_strategy - - if hasattr(config, 'content_filter') and config.content_filter != 'none': - features['content_filter'] = config.content_filter - - if hasattr(config, 'markdown_generation') and config.markdown_generation: - features['markdown_generation'] = True - - if hasattr(config, 'adaptive_crawling') and config.adaptive_crawling: - features['adaptive_crawling'] = True - - if hasattr(config, 'virtual_scrolling') and config.virtual_scrolling: - features['virtual_scrolling'] = True - - if hasattr(config, 'link_analysis') and config.link_analysis: - features['link_analysis'] = True - - return features - - -@router.post("/extract-tables", response_model=Dict[str, Any]) -async def extract_tables_from_html( - request: Dict[str, Any], - api_key_id: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -): - """ - Extract tables from HTML content using advanced table extraction strategies. - - Request body: - { - "html_content": "HTML content", - "base_url": "https://example.com", # optional - "strategy": "default|llm|smart", # optional, default: "default" - "config": {} # optional strategy config - } - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - html_content = request.get("html_content", "") - if not html_content: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="html_content is required" - ) - - base_url = request.get("base_url", "") - strategy = request.get("strategy", "default") - config = request.get("config", {}) - - # Get enhanced scraping service - enhanced_scraper = await get_enhanced_scraping_service() - - # Extract tables - tables = await enhanced_scraper.extract_tables( - html_content=html_content, - base_url=base_url, - strategy=strategy, - config=config - ) - - processing_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) - - response = { - "request_id": request_id, - "tables": tables, - "metadata": { - "total_tables": len(tables), - "processing_time_ms": processing_time_ms, - "strategy_used": strategy - } - } - - logger.info( - "table_extraction_completed", - request_id=request_id, - tables_found=len(tables), - processing_time_ms=processing_time_ms - ) - - return response - - except Exception as e: - logger.error("table_extraction_failed", request_id=request_id, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Table extraction failed: {str(e)}" - ) - - -@router.post("/chunk-content", response_model=Dict[str, Any]) -async def chunk_text_content( - request: Dict[str, Any], - api_key_id: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -): - """ - Chunk text content using various chunking strategies. - - Request body: - { - "text": "Text content to chunk", - "strategy": "paragraph|sentence|fixed|topic|hybrid", # optional, default: "paragraph" - "config": {} # optional strategy config - } - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - text = request.get("text", "") - if not text: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="text is required" - ) - - strategy = request.get("strategy", "paragraph") - config = request.get("config", {}) - - # Get enhanced scraping service - enhanced_scraper = await get_enhanced_scraping_service() - - # Chunk content - chunks = await enhanced_scraper.chunk_content( - text=text, - strategy=strategy, - config=config - ) - - processing_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) - - response = { - "request_id": request_id, - "chunks": chunks, - "metadata": { - "total_chunks": len(chunks), - "original_length": len(text), - "avg_chunk_length": sum(len(chunk) for chunk in chunks) / len(chunks) if chunks else 0, - "processing_time_ms": processing_time_ms, - "strategy_used": strategy - } - } - - logger.info( - "content_chunking_completed", - request_id=request_id, - chunks_created=len(chunks), - processing_time_ms=processing_time_ms - ) - - return response - - except Exception as e: - logger.error("content_chunking_failed", request_id=request_id, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Content chunking failed: {str(e)}" - ) - - -@router.post("/discover-urls", response_model=Dict[str, Any]) -async def discover_urls_from_source( - request: Dict[str, Any], - api_key_id: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -): - """ - Discover URLs from various sources (sitemaps, crawling). - - Request body: - { - "base_url": "https://example.com", - "source": "sitemap|cc|crawl", # optional, default: "sitemap" - "max_urls": 100, # optional - "pattern": "regex_pattern", # optional - "query": "search_query" # optional for relevance scoring - } - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - base_url = request.get("base_url", "") - if not base_url: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="base_url is required" - ) - - config = { - "source": request.get("source", "sitemap"), - "max_urls": request.get("max_urls", 100), - "pattern": request.get("pattern"), - "query": request.get("query"), - "score_threshold": request.get("score_threshold", 0.0) - } - - # Get enhanced scraping service - enhanced_scraper = await get_enhanced_scraping_service() - - # Discover URLs - discovered_urls = await enhanced_scraper.discover_urls( - base_url=base_url, - config=config - ) - - processing_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) - - response = { - "request_id": request_id, - "discovered_urls": discovered_urls, - "metadata": { - "total_urls": len(discovered_urls), - "base_url": base_url, - "source": config["source"], - "processing_time_ms": processing_time_ms - } - } - - logger.info( - "url_discovery_completed", - request_id=request_id, - urls_discovered=len(discovered_urls), - processing_time_ms=processing_time_ms - ) - - return response - - except Exception as e: - logger.error("url_discovery_failed", request_id=request_id, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"URL discovery failed: {str(e)}" - ) - - -@router.get("/performance", response_model=Dict[str, Any]) -async def get_performance_metrics( - api_key_id: ApiKeyDep, - settings: SettingsDep -): - """Get comprehensive performance metrics for the enhanced scraping system.""" - try: - enhanced_scraper = await get_enhanced_scraping_service() - performance_report = await enhanced_scraper.get_performance_report() - - return { - "timestamp": datetime.utcnow().isoformat(), - "performance_metrics": performance_report - } - - except Exception as e: - logger.error("performance_metrics_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to get performance metrics: {str(e)}" - ) diff --git a/apps/backend/app/api/v1/search.py b/apps/backend/app/api/v1/search.py deleted file mode 100644 index cb936e6..0000000 --- a/apps/backend/app/api/v1/search.py +++ /dev/null @@ -1,495 +0,0 @@ -""" -Search and scraping API endpoints. -""" -import asyncio -import uuid -from typing import List, Dict, Any -from datetime import datetime -from fastapi import APIRouter, HTTPException, BackgroundTasks, Request, status -import httpx -from fastapi.responses import JSONResponse, Response -import structlog - -from app.models.requests import UnQuestRequest, BatchSearchRequest, ScrapingConfig -from app.models.responses import ( - UnQuestResponse, AsyncTaskResponse, BatchSearchResponse, - SearchResult, SearchMetadata, EnginesListResponse, HealthResponse, ServiceHealth -) -from app.api.dependencies import ( - ApiKeyDep, SettingsDep, DatabaseDep, SearxngDep, - ScraperDep, CacheDep, ClientInfoDep -) -from app.workers.tasks import process_async_search_scrape - -logger = structlog.get_logger(__name__) - -router = APIRouter(prefix="/search", tags=["search"]) - - -@router.post("/", response_model=UnQuestResponse) -async def search_and_scrape( - request_data: UnQuestRequest, - request: Request, - background_tasks: BackgroundTasks, - api_key_id: ApiKeyDep, - settings: SettingsDep, - db: DatabaseDep, - searxng: SearxngDep, - scraper: ScraperDep, - cache: CacheDep, - client_info: ClientInfoDep -): - """ - Main search and scrape endpoint. - - Performs web search using SearXNG and optionally scrapes content from results. - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - # Generate cache key - cache_key = cache.generate_cache_key(request_data) - - # Check cache if enabled - if request_data.cache_ttl > 0: - cached_response = await cache.get_search_results(cache_key) - if cached_response: - cached_response.request_id = request_id - - # Log request with cached response - await db.log_search_request( - request_data.dict(), - cached_response, - api_key_id, - client_info["client_ip"], - client_info["user_agent"] - ) - - return cached_response - - # Handle async mode - if request_data.async_mode: - if not request_data.webhook_url: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="webhook_url is required for async mode" - ) - - # Create scraping job - job = await db.create_scraping_job( - urls=[], # Will be populated after search - config=request_data.dict(), - webhook_url=str(request_data.webhook_url) - ) - - # Queue async task - task = process_async_search_scrape.delay( - job_id=job.job_id, - request_data=request_data.dict() - ) - - # Update job with task ID - await db.update_scraping_job( - job.job_id, - status="processing", - task_id=task.id - ) - - return AsyncTaskResponse( - task_id=job.job_id, - status="processing", - created_at=datetime.utcnow(), - webhook_url=request_data.webhook_url, - estimated_completion_seconds=30 - ) - - # Perform search - search_start = asyncio.get_event_loop().time() - - # Convert safe_search to numeric - safe_search_map = {"off": 0, "moderate": 1, "strict": 2} - safe_search_value = safe_search_map.get(request_data.safe_search, 1) - - search_results = await searxng.search( - query=request_data.query, - engines=request_data.engines, - language=request_data.language, - safe_search=safe_search_value, - pageno=1 - ) - - search_time_ms = int((asyncio.get_event_loop().time() - search_start) * 1000) - - # Limit results - search_results = search_results[:request_data.max_results] - - # Scrape content if requested - if request_data.scrape_content and search_results: - scraping_start = asyncio.get_event_loop().time() - - # Extract URLs to scrape - urls_to_scrape = [result.url for result in search_results] - - # Create scraping config - scraping_config = ScrapingConfig( - urls=urls_to_scrape, - selectors=request_data.scrape_selectors, - extract_images=request_data.include_images, - extract_links=request_data.include_links, - javascript_rendering=request_data.js_mode, - js_mode=request_data.js_mode, - response_format=request_data.output_format, - screenshot=request_data.screenshot, - pdf=request_data.pdf, - ) - - # Scrape URLs - scraped_contents = await scraper.scrape_urls( - urls_to_scrape[:10], # Limit concurrent scraping - scraping_config - ) - - # Map scraped content to results - scraped_map = {sc.url: sc for sc in scraped_contents} - - for result in search_results: - if str(result.url) in scraped_map: - result.scraped_content = scraped_map[str(result.url)] - - scraping_time_ms = int((asyncio.get_event_loop().time() - scraping_start) * 1000) - else: - scraping_time_ms = 0 - - # Build response - processing_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) - - response = UnQuestResponse( - search_metadata=SearchMetadata( - query=request_data.query, - engines_used=request_data.engines, - engines_succeeded=request_data.engines, # TODO: Track actual successes - engines_failed=[], - total_results_found=len(search_results), - results_returned=len(search_results), - search_time_ms=search_time_ms - ), - results=search_results, - processing_time_ms=processing_time_ms, - cached=False, - cache_key=cache_key, - total_results=len(search_results), - request_id=request_id - ) - - # Cache response if enabled - if request_data.cache_ttl > 0: - background_tasks.add_task( - cache.set_search_results, - cache_key, - response, - request_data.cache_ttl - ) - - # Log request - background_tasks.add_task( - db.log_search_request, - request_data.dict(), - response, - api_key_id, - client_info["client_ip"], - client_info["user_agent"] - ) - - # If markdown format requested, return text/markdown response with concatenated markdown from scraped contents - if request_data.output_format == "markdown" and request_data.scrape_content: - parts = [] - for r in search_results: - if r.scraped_content and r.scraped_content.text: - header = f"# {r.title}\n{r.url}\n\n" if r.title else f"{r.url}\n\n" - parts.append(header + r.scraped_content.text) - markdown_body = "\n\n---\n\n".join(parts) if parts else "" - return Response(content=markdown_body, media_type="text/markdown") - - return response - - except Exception as e: - logger.error( - "search_scrape_error", - request_id=request_id, - query=request_data.query, - error=str(e), - error_type=type(e).__name__ - ) - - # Log error to database - background_tasks.add_task( - db.log_error, - error_type=type(e).__name__, - error_message=str(e), - request_id=request_id, - endpoint="/search", - method="POST", - client_ip=client_info["client_ip"] - ) - - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Search failed: {str(e)}" - ) - - -@router.post("/batch", response_model=BatchSearchResponse) -async def batch_search( - request_data: BatchSearchRequest, - request: Request, - background_tasks: BackgroundTasks, - api_key_id: ApiKeyDep, - settings: SettingsDep, - db: DatabaseDep, - searxng: SearxngDep, - scraper: ScraperDep, - client_info: ClientInfoDep -): - """ - Batch search endpoint for multiple queries. - - Processes multiple search queries in parallel with optional content scraping. - """ - start_time = asyncio.get_event_loop().time() - batch_id = str(uuid.uuid4()) - - try: - results = {} - errors = {} - - # Create semaphore for parallel request limiting - semaphore = asyncio.Semaphore(request_data.parallel_requests) - - async def search_single_query(query: str) -> List[SearchResult]: - """Search a single query with rate limiting.""" - async with semaphore: - try: - search_results = await searxng.search( - query=query, - engines=request_data.engines, - language="en", - safe_search=1 - ) - - # Limit results per query - limited_results = search_results[:request_data.max_results_per_query] - - # Optional content scraping for batch - if request_data.scrape_content and limited_results: - urls = [r.url for r in limited_results[:3]] # Limit scraping in batch - - scraping_config = ScrapingConfig( - urls=urls, - extract_images=False, - extract_links=False - ) - - scraped_contents = await scraper.scrape_urls(urls, scraping_config) - scraped_map = {sc.url: sc for sc in scraped_contents} - - for result in limited_results: - if str(result.url) in scraped_map: - result.scraped_content = scraped_map[str(result.url)] - - return limited_results - - except Exception as e: - logger.error("batch_search_query_error", query=query, error=str(e)) - errors[query] = str(e) - return [] - - # Execute searches in parallel - tasks = [search_single_query(query) for query in request_data.queries] - search_results = await asyncio.gather(*tasks) - - # Map results to queries - for query, query_results in zip(request_data.queries, search_results): - if query not in errors: - results[query] = query_results - - processing_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) - - # Log batch request - background_tasks.add_task( - db.log_search_request, - { - "query": f"BATCH:{len(request_data.queries)} queries", - "engines": request_data.engines, - "max_results": request_data.max_results_per_query, - "request_id": batch_id - }, - None, - api_key_id, - client_info["client_ip"], - client_info["user_agent"] - ) - - return BatchSearchResponse( - batch_id=batch_id, - queries_processed=len(results), - queries_failed=len(errors), - results=results, - processing_time_ms=processing_time_ms, - errors=errors - ) - - except Exception as e: - logger.error("batch_search_error", batch_id=batch_id, error=str(e)) - - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Batch search failed: {str(e)}" - ) - - -@router.get("/engines", response_model=EnginesListResponse) -async def list_engines( - searxng: SearxngDep, - api_key_id: ApiKeyDep -): - """ - List available search engines. - - Returns information about all configured search engines including their - capabilities and current status. - """ - try: - engines = await searxng.get_available_engines() - - return EnginesListResponse( - engines=engines, - total_engines=len(engines), - enabled_engines=sum(1 for e in engines.values() if e.enabled) - ) - - except Exception as e: - logger.error("list_engines_error", error=str(e)) - - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to list engines: {str(e)}" - ) - - -@router.get("/health", response_model=HealthResponse) -async def health_check( - settings: SettingsDep, - searxng: SearxngDep, - cache: CacheDep, - db: DatabaseDep -): - """ - Health check endpoint. - - Checks the health of all service dependencies. - """ - start_time = datetime.utcnow() - - # Check SearXNG - searxng_health = await searxng.health_check() - - # Check Redis - cache_start = asyncio.get_event_loop().time() - try: - await cache._client.ping() - cache_health = ServiceHealth( - status="healthy", - latency_ms=int((asyncio.get_event_loop().time() - cache_start) * 1000), - last_check=datetime.utcnow() - ) - except Exception as e: - cache_health = ServiceHealth( - status="unhealthy", - latency_ms=int((asyncio.get_event_loop().time() - cache_start) * 1000), - last_check=datetime.utcnow(), - details={"error": str(e)} - ) - - # Check Database - db_start = asyncio.get_event_loop().time() - try: - async with db.get_session() as session: - await session.execute("SELECT 1") - db_health = ServiceHealth( - status="healthy", - latency_ms=int((asyncio.get_event_loop().time() - db_start) * 1000), - last_check=datetime.utcnow() - ) - except Exception as e: - db_health = ServiceHealth( - status="unhealthy", - latency_ms=int((asyncio.get_event_loop().time() - db_start) * 1000), - last_check=datetime.utcnow(), - details={"error": str(e)} - ) - - # Check Puppeteer (if enabled) - puppeteer_health = ServiceHealth( - status="degraded", - latency_ms=0, - last_check=datetime.utcnow(), - details={"enabled": False} - ) - if getattr(settings, 'puppeteer_enabled', False) and settings.puppeteer_service_url: - pupp_start = asyncio.get_event_loop().time() - try: - base = str(settings.puppeteer_service_url).rstrip('/') - async with httpx.AsyncClient(timeout=httpx.Timeout(3.0)) as client: - # Try /health then fallback to / - tried = False - for path in ("/health", "/"): - try: - resp = await client.get(base + path) - resp.raise_for_status() - tried = True - break - except Exception: - continue - status_ok = tried - puppeteer_health = ServiceHealth( - status="healthy" if status_ok else "unhealthy", - latency_ms=int((asyncio.get_event_loop().time() - pupp_start) * 1000), - last_check=datetime.utcnow(), - details={"url": base} - ) - except Exception as e: - puppeteer_health = ServiceHealth( - status="unhealthy", - latency_ms=int((asyncio.get_event_loop().time() - pupp_start) * 1000), - last_check=datetime.utcnow(), - details={"error": str(e), "url": str(settings.puppeteer_service_url)} - ) - - # Determine overall status - services = { - "searxng": searxng_health, - "redis": cache_health, - "database": db_health, - "puppeteer": puppeteer_health - } - - unhealthy_count = sum(1 for s in services.values() if s.status == "unhealthy") - - if unhealthy_count == 0: - overall_status = "healthy" - elif unhealthy_count < len(services): - overall_status = "degraded" - else: - overall_status = "unhealthy" - - uptime_seconds = int((datetime.utcnow() - start_time).total_seconds()) - - return HealthResponse( - status=overall_status, - version=settings.version, - environment=settings.environment, - services=services, - timestamp=datetime.utcnow(), - uptime_seconds=uptime_seconds - ) diff --git a/apps/backend/app/api/v2/__init__.py b/apps/backend/app/api/v2/__init__.py deleted file mode 100644 index 7731211..0000000 --- a/apps/backend/app/api/v2/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# API v2 initialization diff --git a/apps/backend/app/api/v2/advanced_endpoints.py b/apps/backend/app/api/v2/advanced_endpoints.py deleted file mode 100644 index 7ad5d83..0000000 --- a/apps/backend/app/api/v2/advanced_endpoints.py +++ /dev/null @@ -1,1347 +0,0 @@ -""" -Advanced API endpoints with Firecrawl-inspired functionality. - -Provides cutting-edge features including: -- Multi-engine scraping with intelligent fallback -- Multi-provider search with automatic failover -- LLM-powered configuration generation -- Multi-entity extraction with relationship mapping -- Advanced batch processing operations -""" - -import asyncio -import json -import uuid -from typing import List, Dict, Any, Optional, Union -from datetime import datetime -from fastapi import APIRouter, HTTPException, BackgroundTasks, Request, status, Query -from fastapi.responses import JSONResponse, Response -from pydantic import BaseModel, Field -import structlog - -from app.models.requests import ScrapingConfig -from app.models.responses import UnQuestResponse, AsyncTaskResponse, SearchResult -from app.api.dependencies import ( - ApiKeyDep, SettingsDep, DatabaseDep, - CacheDep, ClientInfoDep -) -from app.services.multi_search import get_multi_search_service, SearchOptions -from app.services.multi_engine_scraper import get_multi_engine_service, EngineType -from app.services.llm_configuration import get_llm_config_service, generate_config_from_prompt -from app.services.batch_operations import get_batch_service -from app.services.multi_entity_extraction import ( - get_multi_entity_service, - MultiEntityExtractionRequest, - ExtractionStrategy -) -from app.services.actions_system import get_actions_service, execute_browser_actions -from app.services.website_mapping import get_website_mapper, MapOptions, MapStrategy -from app.services.change_tracking import get_change_tracking_service, ChangeTrackingConfig -from app.services.attributes_extraction import get_attributes_extractor, AttributeExtractionRule, AttributeProcessingType - -logger = structlog.get_logger(__name__) -router = APIRouter(prefix="/v2/advanced", tags=["advanced-features"]) - - -# Request/Response Models - -# Actions System Models -class ActionRequest(BaseModel): - """Single browser action request.""" - type: str = Field(..., description="Action type (wait, click, scroll, write, press, screenshot, scrape, executeJavascript, pdf)") - # Action-specific fields (will be validated by the actions system) - milliseconds: Optional[int] = Field(None, description="Wait time in milliseconds (for wait actions)") - selector: Optional[str] = Field(None, description="CSS selector (for click, scroll, wait actions)") - text: Optional[str] = Field(None, description="Text to write (for write actions)") - key: Optional[str] = Field(None, description="Key to press (for press actions)") - script: Optional[str] = Field(None, description="JavaScript to execute") - fullPage: Optional[bool] = Field(False, description="Full page screenshot") - quality: Optional[int] = Field(None, ge=1, le=100, description="Screenshot quality") - direction: Optional[str] = Field("down", description="Scroll direction") - all: Optional[bool] = Field(False, description="Click all matching elements") - -class BrowserActionsRequest(BaseModel): - """Request for browser actions sequence.""" - url: str = Field(..., description="URL to navigate to") - actions: List[ActionRequest] = Field(..., description="Sequence of actions to perform") - browser_options: Optional[Dict[str, Any]] = Field(None, description="Browser configuration options") - -# Website Mapping Models -class WebsiteMapRequest(BaseModel): - """Request for website mapping.""" - url: str = Field(..., description="Website URL to map") - strategy: str = Field("combined", description="Mapping strategy (sitemap_only, search_engine, combined, crawl_based)") - limit: int = Field(1000, ge=1, le=10000, description="Maximum URLs to return") - include_subdomains: bool = Field(True, description="Include subdomains") - allow_external_links: bool = Field(False, description="Allow external links") - search_query: Optional[str] = Field(None, description="Search query for filtering") - ignore_sitemap: bool = Field(False, description="Ignore sitemap") - filter_by_path: bool = Field(True, description="Filter by URL path") - timeout: int = Field(30, ge=5, le=120, description="Request timeout") - max_depth: int = Field(3, ge=1, le=10, description="Maximum crawl depth") - -# Change Tracking Models -class ChangeTrackingRequest(BaseModel): - """Request for change tracking.""" - url: str = Field(..., description="URL to track changes for") - tag: Optional[str] = Field(None, description="Tag for grouping tracked content") - threshold: float = Field(0.05, ge=0.0, le=1.0, description="Minimum change percentage to trigger notification") - compare_text: bool = Field(True, description="Compare text content") - compare_html: bool = Field(True, description="Compare HTML content") - compare_metadata: bool = Field(True, description="Compare metadata") - notification_webhook: Optional[str] = Field(None, description="Webhook URL for notifications") - store_history: bool = Field(True, description="Store change history") - -# Attributes Extraction Models -class AttributeExtractionRuleRequest(BaseModel): - """Single attribute extraction rule.""" - selector: str = Field(..., description="CSS selector") - attribute: str = Field(..., description="HTML attribute to extract") - processing: str = Field("cleaned", description="Processing type (raw, cleaned, urls_resolved, numeric, boolean, list)") - filter_empty: bool = Field(True, description="Filter empty values") - filter_duplicates: bool = Field(True, description="Filter duplicate values") - limit: Optional[int] = Field(None, description="Maximum elements to process") - transform: Optional[str] = Field(None, description="Transform function") - validation_pattern: Optional[str] = Field(None, description="Validation regex pattern") - -class AttributesExtractionRequest(BaseModel): - """Request for attributes extraction.""" - url: str = Field(..., description="URL to extract attributes from") - rules: List[AttributeExtractionRuleRequest] = Field(..., description="Extraction rules") - base_url: Optional[str] = Field(None, description="Base URL for URL resolution") - include_element_context: bool = Field(True, description="Include element context") - max_elements_per_selector: int = Field(1000, ge=1, le=10000, description="Max elements per selector") - resolve_relative_urls: bool = Field(True, description="Resolve relative URLs") - -# Combined Request Models -class AdvancedScrapeRequest(BaseModel): - """Advanced scraping request with all features.""" - url: str = Field(..., description="URL to scrape") - actions: Optional[List[ActionRequest]] = Field(None, description="Browser actions to perform before scraping") - change_tracking: Optional[ChangeTrackingRequest] = Field(None, description="Change tracking configuration") - attributes_extraction: Optional[AttributesExtractionRequest] = Field(None, description="Attributes extraction configuration") - scraping_config: Optional[ScrapingConfig] = Field(None, description="Scraping configuration") - engine: Optional[str] = Field(None, description="Preferred scraping engine") -class MultiProviderSearchRequest(BaseModel): - """Request for multi-provider search.""" - query: str = Field(..., description="Search query") - num_results: int = Field(10, ge=1, le=100, description="Number of results") - lang: str = Field("en", description="Language code") - country: str = Field("us", description="Country code") - location: Optional[str] = Field(None, description="Location for search") - tbs: Optional[str] = Field(None, description="Time-based search filter") - filter: Optional[str] = Field(None, description="Search filter") - advanced: bool = Field(False, description="Enable advanced search") - scrape_results: bool = Field(False, description="Scrape search results") - scrape_config: Optional[Dict[str, Any]] = Field(None, description="Scraping configuration") - - -class MultiEngineScrapingRequest(BaseModel): - """Request for multi-engine scraping.""" - urls: List[str] = Field(..., description="URLs to scrape") - preferred_engine: Optional[str] = Field(None, description="Preferred scraping engine") - required_capabilities: List[str] = Field(default_factory=list, description="Required engine capabilities") - config: Optional[ScrapingConfig] = Field(None, description="Scraping configuration") - timeout: int = Field(30, ge=5, le=300, description="Timeout per URL in seconds") - - -class LLMConfigurationRequest(BaseModel): - """Request for LLM-powered configuration generation.""" - prompt: str = Field(..., description="Natural language configuration prompt") - config_type: str = Field(..., description="Type of configuration (crawler|extraction|filter|search)") - context: Optional[Dict[str, Any]] = Field(None, description="Additional context") - - -class BatchOperationRequest(BaseModel): - """Request for batch operations.""" - operation_type: str = Field(..., description="Type of batch operation (scrape|search|extract)") - urls: List[str] = Field(..., description="URLs or queries to process") - config: Optional[Dict[str, Any]] = Field(None, description="Operation configuration") - priority: int = Field(10, ge=1, le=100, description="Job priority (lower = higher priority)") - webhook_url: Optional[str] = Field(None, description="Webhook URL for status updates") - metadata: Optional[Dict[str, Any]] = Field(None, description="Additional metadata") - - -class MultiEntityExtractionRequestModel(BaseModel): - """Request for multi-entity extraction.""" - urls: List[str] = Field(..., description="URLs to extract entities from") - schema: Dict[str, Any] = Field(..., description="JSON schema for extraction") - extraction_strategy: str = Field("linked_entities", description="Extraction strategy") - max_related_urls: int = Field(50, ge=1, le=200, description="Maximum related URLs to discover") - similarity_threshold: float = Field(0.7, ge=0.0, le=1.0, description="Similarity threshold") - cross_validate: bool = Field(True, description="Enable cross-validation") - follow_links: bool = Field(True, description="Follow links to discover related URLs") - max_depth: int = Field(2, ge=1, le=5, description="Maximum crawl depth") - - -# API Endpoints - -@router.post("/search/multi-provider") -async def multi_provider_search( - request_data: MultiProviderSearchRequest, - request: Request, - api_key_id: ApiKeyDep, - settings: SettingsDep, - db: DatabaseDep, - cache: CacheDep, - client_info: ClientInfoDep -): - """ - Advanced multi-provider search with intelligent fallback. - - Uses multiple search providers with automatic failover: - - Fire Engine (if available) - - Serper API (if configured) - - SearchAPI (if configured) - - SearXNG (if configured) - - Google (fallback) - - Optional result scraping with multi-engine support. - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - logger.info("multi_provider_search_started", - request_id=request_id, - query=request_data.query, - scrape_results=request_data.scrape_results) - - # Perform multi-provider search - search_service = await get_multi_search_service() - search_options = SearchOptions( - query=request_data.query, - num_results=request_data.num_results, - lang=request_data.lang, - country=request_data.country, - location=request_data.location, - tbs=request_data.tbs, - filter=request_data.filter, - advanced=request_data.advanced - ) - - search_results = await search_service.search(search_options) - - # Optionally scrape search results - scraped_results = [] - if request_data.scrape_results and search_results: - urls_to_scrape = [result.url for result in search_results] - - # Use multi-engine scraping for results - engine_service = await get_multi_engine_service() - - scrape_config = ScrapingConfig(**(request_data.scrape_config or {})) - - for url in urls_to_scrape: - try: - scrape_result = await engine_service.scrape(url, scrape_config) - if scrape_result.success: - scraped_results.append({ - "url": url, - "content": scrape_result.content.dict(), - "engine_used": scrape_result.engine_used.value, - "processing_time": scrape_result.processing_time - }) - except Exception as e: - logger.warning("scraping_search_result_failed", url=url, error=str(e)) - - processing_time = asyncio.get_event_loop().time() - start_time - - # Prepare response - response_data = { - "request_id": request_id, - "query": request_data.query, - "search_results": [result.dict() for result in search_results], - "scraped_results": scraped_results, - "metadata": { - "total_results": len(search_results), - "scraped_count": len(scraped_results), - "processing_time_ms": int(processing_time * 1000), - "search_providers_available": await search_service.get_provider_stats() - } - } - - logger.info("multi_provider_search_completed", - request_id=request_id, - results_count=len(search_results), - scraped_count=len(scraped_results), - processing_time=processing_time) - - return response_data - - except Exception as e: - logger.error("multi_provider_search_failed", request_id=request_id, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Multi-provider search failed: {str(e)}" - ) - - -@router.post("/scrape/multi-engine") -async def multi_engine_scraping( - request_data: MultiEngineScrapingRequest, - api_key_id: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -): - """ - Advanced multi-engine scraping with intelligent engine selection. - - Automatically selects the best scraping engine based on: - - Content type detection - - Required capabilities - - Engine availability and performance - - Fallback strategies - - Available engines: - - Index (cached content) - - Fire Engine variants (Chrome CDP, Playwright, TLS Client) - - Playwright service - - Basic fetch (fallback) - - PDF/DOCX processors - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - logger.info("multi_engine_scraping_started", - request_id=request_id, - urls=len(request_data.urls), - preferred_engine=request_data.preferred_engine) - - engine_service = await get_multi_engine_service() - - # Convert preferred engine string to enum if provided - preferred_engine = None - if request_data.preferred_engine: - try: - preferred_engine = EngineType(request_data.preferred_engine) - except ValueError: - logger.warning("invalid_preferred_engine", engine=request_data.preferred_engine) - - # Process each URL - results = [] - for url in request_data.urls: - try: - scrape_result = await engine_service.scrape( - url=url, - config=request_data.config or ScrapingConfig(urls=[]), - preferred_engine=preferred_engine, - required_capabilities=request_data.required_capabilities - ) - - results.append({ - "url": url, - "success": scrape_result.success, - "content": scrape_result.content.dict() if scrape_result.success else None, - "engine_used": scrape_result.engine_used.value, - "processing_time": scrape_result.processing_time, - "attempts": scrape_result.attempts, - "error": scrape_result.error - }) - - except Exception as e: - logger.error("url_scraping_failed", url=url, error=str(e)) - results.append({ - "url": url, - "success": False, - "content": None, - "engine_used": "none", - "processing_time": 0, - "attempts": 0, - "error": str(e) - }) - - processing_time = asyncio.get_event_loop().time() - start_time - successful_results = [r for r in results if r["success"]] - - # Get engine statistics - engine_stats = await engine_service.get_engine_stats() - - response_data = { - "request_id": request_id, - "results": results, - "metadata": { - "total_urls": len(request_data.urls), - "successful_scrapes": len(successful_results), - "failed_scrapes": len(results) - len(successful_results), - "processing_time_ms": int(processing_time * 1000), - "engine_statistics": engine_stats - } - } - - logger.info("multi_engine_scraping_completed", - request_id=request_id, - successful=len(successful_results), - failed=len(results) - len(successful_results), - processing_time=processing_time) - - return response_data - - except Exception as e: - logger.error("multi_engine_scraping_failed", request_id=request_id, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Multi-engine scraping failed: {str(e)}" - ) - - -@router.post("/config/generate") -async def generate_configuration( - request_data: LLMConfigurationRequest, - api_key_id: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -): - """ - Generate configuration from natural language using LLM. - - Converts natural language descriptions into structured configurations: - - Crawler options and settings - - Data extraction schemas - - Content filtering rules - - Search strategies - - Example prompts: - - "Crawl a blog site and extract only the article pages" - - "Extract product information including name, price, and reviews" - - "Filter content to only include technical articles about AI" - - "Search for recent news articles from reliable sources" - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - logger.info("llm_configuration_started", - request_id=request_id, - config_type=request_data.config_type, - prompt_length=len(request_data.prompt)) - - # Generate configuration using LLM - config = await generate_config_from_prompt( - prompt=request_data.prompt, - config_type=request_data.config_type, - context=request_data.context - ) - - processing_time = asyncio.get_event_loop().time() - start_time - - # Get LLM service stats - llm_service = await get_llm_config_service() - usage_stats = await llm_service.get_usage_stats() - - response_data = { - "request_id": request_id, - "config_type": request_data.config_type, - "generated_config": config, - "metadata": { - "processing_time_ms": int(processing_time * 1000), - "prompt_length": len(request_data.prompt), - "config_fields_generated": len(config) if isinstance(config, dict) else 0, - "llm_usage_stats": usage_stats - } - } - - logger.info("llm_configuration_completed", - request_id=request_id, - config_fields=len(config) if isinstance(config, dict) else 0, - processing_time=processing_time) - - return response_data - - except Exception as e: - logger.error("llm_configuration_failed", request_id=request_id, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"LLM configuration generation failed: {str(e)}" - ) - - -@router.post("/batch/submit") -async def submit_batch_operation( - request_data: BatchOperationRequest, - api_key_id: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -): - """ - Submit advanced batch operation. - - Supports various batch operations: - - Batch scraping with intelligent resource management - - Batch search across multiple providers - - Batch extraction with entity linking - - Features: - - Intelligent job scheduling and prioritization - - Progress tracking and status updates - - Error handling and retry logic - - Webhook notifications - - Job pause/resume/cancel capabilities - """ - try: - logger.info("batch_operation_submitted", - operation_type=request_data.operation_type, - urls=len(request_data.urls), - priority=request_data.priority) - - batch_service = await get_batch_service() - - # Submit appropriate batch operation - if request_data.operation_type == "scrape": - # Convert config to ScrapingConfig if provided - scrape_config = None - if request_data.config: - scrape_config = ScrapingConfig(**request_data.config) - - job_id = await batch_service.submit_batch_scrape( - urls=request_data.urls, - config=scrape_config, - priority=request_data.priority, - webhook_url=request_data.webhook_url, - metadata=request_data.metadata - ) - - elif request_data.operation_type == "search": - job_id = await batch_service.submit_batch_search( - queries=request_data.urls, # Using URLs field for queries - search_config=request_data.config, - priority=request_data.priority, - webhook_url=request_data.webhook_url, - metadata=request_data.metadata - ) - - else: - raise ValueError(f"Unsupported operation type: {request_data.operation_type}") - - # Get initial job status - job_status = await batch_service.get_job_status(job_id) - - response_data = { - "job_id": job_id, - "operation_type": request_data.operation_type, - "status": job_status.status.value if job_status else "unknown", - "urls_count": len(request_data.urls), - "priority": request_data.priority, - "webhook_url": request_data.webhook_url, - "estimated_completion": None, # Will be updated as job progresses - "created_at": datetime.utcnow().isoformat() - } - - logger.info("batch_operation_accepted", - job_id=job_id, - operation_type=request_data.operation_type) - - return response_data - - except Exception as e: - logger.error("batch_operation_submission_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Batch operation submission failed: {str(e)}" - ) - - -@router.get("/batch/{job_id}/status") -async def get_batch_status( - job_id: str, - api_key_id: ApiKeyDep, - settings: SettingsDep -): - """Get status of batch operation.""" - try: - batch_service = await get_batch_service() - job_status = await batch_service.get_job_status(job_id) - - if not job_status: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Job {job_id} not found" - ) - - response_data = { - "job_id": job_status.job_id, - "status": job_status.status.value, - "progress": { - "total_urls": job_status.progress.total_urls if job_status.progress else 0, - "completed_urls": job_status.progress.completed_urls if job_status.progress else 0, - "failed_urls": job_status.progress.failed_urls if job_status.progress else 0, - "skipped_urls": job_status.progress.skipped_urls if job_status.progress else 0, - "completion_percentage": job_status.progress.completion_percentage if job_status.progress else 0, - "estimated_completion": job_status.progress.estimated_completion.isoformat() if job_status.progress and job_status.progress.estimated_completion else None, - "current_url": job_status.progress.current_url if job_status.progress else None - }, - "results_count": len(job_status.results), - "errors_count": len(job_status.errors), - "metadata": job_status.metadata, - "created_at": job_status.created_at.isoformat() if job_status.created_at else None, - "updated_at": job_status.updated_at.isoformat() if job_status.updated_at else None - } - - return response_data - - except HTTPException: - raise - except Exception as e: - logger.error("batch_status_failed", job_id=job_id, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to get batch status: {str(e)}" - ) - - -@router.post("/batch/{job_id}/control") -async def control_batch_operation( - job_id: str, - action: str = Query(..., description="Action to perform (pause|resume|cancel)"), - api_key_id: ApiKeyDep, - settings: SettingsDep -): - """Control batch operation (pause, resume, cancel).""" - try: - batch_service = await get_batch_service() - - result = False - if action == "pause": - result = await batch_service.pause_job(job_id) - elif action == "resume": - result = await batch_service.resume_job(job_id) - elif action == "cancel": - result = await batch_service.cancel_job(job_id) - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid action: {action}. Use pause, resume, or cancel." - ) - - if not result: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Job {job_id} not found or action not allowed" - ) - - return { - "job_id": job_id, - "action": action, - "success": result, - "timestamp": datetime.utcnow().isoformat() - } - - except HTTPException: - raise - except Exception as e: - logger.error("batch_control_failed", job_id=job_id, action=action, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to control batch operation: {str(e)}" - ) - - -@router.post("/extract/multi-entity") -async def multi_entity_extraction( - request_data: MultiEntityExtractionRequestModel, - api_key_id: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -): - """ - Advanced multi-entity extraction with relationship mapping. - - Performs sophisticated cross-URL data extraction: - - Discovers related URLs through various strategies - - Extracts entities using multiple methods (LLM, regex, CSS) - - Maps relationships between entities - - Validates data through cross-referencing - - Provides comprehensive entity analytics - - Extraction strategies: - - linked_entities: Extract entities and find related URLs - - hierarchical: Follow hierarchical relationships - - semantic_similarity: Group by semantic similarity - - temporal_sequence: Time-based entity relationships - - cross_reference: Cross-reference validation - """ - start_time = asyncio.get_event_loop().time() - request_id = str(uuid.uuid4()) - - try: - logger.info("multi_entity_extraction_started", - request_id=request_id, - urls=len(request_data.urls), - strategy=request_data.extraction_strategy) - - # Convert string strategy to enum - try: - strategy = ExtractionStrategy(request_data.extraction_strategy) - except ValueError: - strategy = ExtractionStrategy.LINKED_ENTITIES - - # Create extraction request - extraction_request = MultiEntityExtractionRequest( - urls=request_data.urls, - schema=request_data.schema, - extraction_strategy=strategy, - max_related_urls=request_data.max_related_urls, - similarity_threshold=request_data.similarity_threshold, - cross_validate=request_data.cross_validate, - follow_links=request_data.follow_links, - max_depth=request_data.max_depth - ) - - # Perform extraction - extraction_service = await get_multi_entity_service() - result = await extraction_service.extract_multi_entity(extraction_request) - - processing_time = asyncio.get_event_loop().time() - start_time - - # Prepare response - response_data = { - "request_id": result.request_id, - "success": result.success, - "entities": [ - { - "id": entity.id, - "type": entity.entity_type, - "value": entity.value, - "confidence": entity.confidence, - "source_url": entity.source_url, - "extraction_method": entity.extraction_method, - "context": entity.context, - "attributes": entity.attributes, - "related_entities": entity.related_entities - } - for entity in result.entities - ], - "relationships": [ - { - "source_url": rel.source_url, - "target_url": rel.target_url, - "relation_type": rel.relation_type, - "confidence": rel.confidence, - "evidence": rel.evidence - } - for rel in result.relations - ], - "discovered_urls": result.discovered_urls, - "validation_results": result.validation_results, - "extraction_metadata": result.extraction_metadata, - "processing_time": result.processing_time, - "errors": result.errors - } - - logger.info("multi_entity_extraction_completed", - request_id=request_id, - entities_extracted=len(result.entities), - relationships_found=len(result.relations), - urls_discovered=len(result.discovered_urls), - processing_time=processing_time) - - return response_data - - except Exception as e: - logger.error("multi_entity_extraction_failed", request_id=request_id, error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Multi-entity extraction failed: {str(e)}" - ) - - -@router.get("/stats/comprehensive") -async def get_comprehensive_stats( - api_key_id: ApiKeyDep, - settings: SettingsDep -): - """Get comprehensive statistics for all advanced services.""" - try: - stats = {} - - # Multi-search service stats - try: - search_service = await get_multi_search_service() - stats["multi_search"] = await search_service.get_provider_stats() - except Exception as e: - stats["multi_search"] = {"error": str(e)} - - # Multi-engine scraping stats - try: - engine_service = await get_multi_engine_service() - stats["multi_engine_scraping"] = await engine_service.get_engine_stats() - except Exception as e: - stats["multi_engine_scraping"] = {"error": str(e)} - - # LLM configuration stats - try: - llm_service = await get_llm_config_service() - stats["llm_configuration"] = await llm_service.get_usage_stats() - except Exception as e: - stats["llm_configuration"] = {"error": str(e)} - - # Batch operations stats - try: - batch_service = await get_batch_service() - stats["batch_operations"] = await batch_service.get_service_stats() - except Exception as e: - stats["batch_operations"] = {"error": str(e)} - - # Multi-entity extraction stats - try: - extraction_service = await get_multi_entity_service() - stats["multi_entity_extraction"] = await extraction_service.get_extraction_stats() - except Exception as e: - stats["multi_entity_extraction"] = {"error": str(e)} - - # Actions system stats - try: - actions_service = await get_actions_service() - stats["actions_system"] = await actions_service.get_actions_stats() - except Exception as e: - stats["actions_system"] = {"error": str(e)} - - # Website mapping stats - try: - mapping_service = await get_website_mapper() - stats["website_mapping"] = await mapping_service.get_mapping_stats() - except Exception as e: - stats["website_mapping"] = {"error": str(e)} - - # Change tracking stats - try: - tracking_service = await get_change_tracking_service() - stats["change_tracking"] = await tracking_service.get_tracking_stats() - except Exception as e: - stats["change_tracking"] = {"error": str(e)} - - # Attributes extraction stats - try: - attributes_service = await get_attributes_extractor() - stats["attributes_extraction"] = await attributes_service.get_extraction_stats() - except Exception as e: - stats["attributes_extraction"] = {"error": str(e)} - - return { - "timestamp": datetime.utcnow().isoformat(), - "services": stats - } - - except Exception as e: - logger.error("comprehensive_stats_failed", error=str(e)) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to get comprehensive stats: {str(e)}" - ) - - -@router.get("/health/advanced") -async def advanced_health_check(): - """Health check for advanced services.""" - try: - health_status = { - "status": "healthy", - "timestamp": datetime.utcnow().isoformat(), - "services": {} - } - - # Check each service - services_to_check = [ - ("multi_search", get_multi_search_service), - ("multi_engine_scraping", get_multi_engine_service), - ("llm_configuration", get_llm_config_service), - ("batch_operations", get_batch_service), - ("multi_entity_extraction", get_multi_entity_service), - ("actions_system", get_actions_service), - ("website_mapping", get_website_mapper), - ("change_tracking", get_change_tracking_service), - ("attributes_extraction", get_attributes_extractor) - ] - - for service_name, service_getter in services_to_check: - try: - service = await service_getter() - health_status["services"][service_name] = { - "status": "healthy", - "initialized": service is not None - } - except Exception as e: - health_status["services"][service_name] = { - "status": "unhealthy", - "error": str(e) - } - health_status["status"] = "degraded" - - # Return appropriate status code - status_code = 200 if health_status["status"] == "healthy" else 503 - - return JSONResponse( - content=health_status, - status_code=status_code - ) - - except Exception as e: - return JSONResponse( - content={ - "status": "unhealthy", - "error": str(e), - "timestamp": datetime.utcnow().isoformat() - }, - status_code=503 - ) - - -@router.post("/actions/execute", summary="Execute Browser Actions", status_code=status.HTTP_200_OK) -async def execute_browser_actions_endpoint( - request: BrowserActionsRequest, - api_key: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -) -> UnQuestResponse: - """ - Execute a sequence of browser actions on a webpage. - - Supports all Firecrawl action types: - - wait: Wait for time or element - - click: Click elements - - scroll: Scroll page - - write: Type text - - press: Press keys - - screenshot: Capture screenshots - - scrape: Extract page content - - executeJavascript: Run JavaScript - - pdf: Generate PDF - """ - try: - logger.info("browser_actions_execution_started", - url=request.url, - actions_count=len(request.actions), - client_info=client_info) - - # Convert request actions to dict format - actions_dicts = [action.dict(exclude_none=True) for action in request.actions] - - # Execute actions - result = await execute_browser_actions( - url=request.url, - actions=actions_dicts, - browser_options=request.browser_options - ) - - if result.success: - return UnQuestResponse( - success=True, - data={ - "actions_results": [ - { - "action_type": ar.action_type, - "success": ar.success, - "data": ar.data, - "execution_time_ms": ar.execution_time_ms, - "error": ar.error - } - for ar in result.actions_results - ], - "screenshots": result.screenshots, - "scrapes": result.scrapes, - "javascript_returns": result.javascript_returns, - "pdfs": result.pdfs, - "total_execution_time_ms": result.total_execution_time_ms - }, - message=f"Executed {len(request.actions)} actions successfully" - ) - else: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f"Actions execution failed: {result.error}" - ) - - except Exception as e: - logger.error("browser_actions_execution_failed", - url=request.url, - error=str(e), - client_info=client_info) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Actions execution failed: {str(e)}" - ) - - -@router.post("/map/website", summary="Map Website URLs", status_code=status.HTTP_200_OK) -async def map_website_endpoint( - request: WebsiteMapRequest, - api_key: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -) -> UnQuestResponse: - """ - Discover all URLs from a website using multiple strategies. - - Strategies: - - sitemap_only: Use XML sitemaps only - - search_engine: Use search engine queries - - combined: Use both sitemaps and search engines - - crawl_based: Use web crawling - """ - try: - logger.info("website_mapping_started", - url=request.url, - strategy=request.strategy, - limit=request.limit, - client_info=client_info) - - mapper = await get_website_mapper() - - # Create mapping options - map_options = MapOptions( - strategy=MapStrategy(request.strategy), - limit=request.limit, - include_subdomains=request.include_subdomains, - allow_external_links=request.allow_external_links, - search_query=request.search_query, - ignore_sitemap=request.ignore_sitemap, - filter_by_path=request.filter_by_path, - timeout=request.timeout, - max_depth=request.max_depth - ) - - # Execute mapping - result = await mapper.map_website(request.url, map_options) - - if result.success: - return UnQuestResponse( - success=True, - data={ - "base_url": result.base_url, - "discovered_urls": [ - { - "url": du.url, - "source": du.source, - "title": du.title, - "description": du.description, - "last_modified": du.last_modified, - "priority": du.priority, - "depth": du.depth - } - for du in result.discovered_urls - ], - "total_urls": result.total_urls, - "sources_breakdown": result.sources_breakdown, - "processing_time_ms": result.processing_time_ms, - "metadata": result.metadata - }, - message=f"Discovered {result.total_urls} URLs from {request.url}" - ) - else: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f"Website mapping failed: {result.error}" - ) - - except Exception as e: - logger.error("website_mapping_failed", - url=request.url, - error=str(e), - client_info=client_info) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Website mapping failed: {str(e)}" - ) - - -@router.post("/track/changes", summary="Track Content Changes", status_code=status.HTTP_200_OK) -async def track_content_changes_endpoint( - request: ChangeTrackingRequest, - api_key: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -) -> UnQuestResponse: - """ - Track changes in website content over time. - - Features: - - Content comparison and diff generation - - Change percentage calculation - - Historical tracking - - Webhook notifications - """ - try: - logger.info("change_tracking_started", - url=request.url, - tag=request.tag, - threshold=request.threshold, - client_info=client_info) - - tracking_service = await get_change_tracking_service() - - # Create tracking configuration - config = ChangeTrackingConfig( - tag=request.tag, - threshold=request.threshold, - compare_text=request.compare_text, - compare_html=request.compare_html, - compare_metadata=request.compare_metadata, - notification_webhook=request.notification_webhook, - store_history=request.store_history - ) - - # Execute change tracking - result = await tracking_service.track_content_changes(request.url, config) - - if result.success: - # Convert scraped content to dict - scraped_content_dict = result.scraped_content.dict() - - return UnQuestResponse( - success=True, - data={ - "url": result.url, - "change_tracking": { - "previous_scrape_at": result.tracking_data.previous_scrape_at.isoformat() if result.tracking_data.previous_scrape_at else None, - "change_status": result.tracking_data.change_status.value, - "visibility": result.tracking_data.visibility.value, - "change_percentage": result.tracking_data.change_percentage, - "significant_changes": result.tracking_data.significant_changes, - "diff": { - "text": result.tracking_data.diff.text_diff if result.tracking_data.diff else "", - "json": result.tracking_data.diff.json_diff if result.tracking_data.diff else {} - } if result.tracking_data.diff else None - }, - "scraped_content": scraped_content_dict, - "processing_time_ms": result.processing_time_ms - }, - message=f"Change tracking completed for {request.url} - Status: {result.tracking_data.change_status.value}" - ) - else: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f"Change tracking failed: {result.error}" - ) - - except Exception as e: - logger.error("change_tracking_failed", - url=request.url, - error=str(e), - client_info=client_info) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Change tracking failed: {str(e)}" - ) - - -@router.post("/extract/attributes", summary="Extract HTML Attributes", status_code=status.HTTP_200_OK) -async def extract_attributes_endpoint( - request: AttributesExtractionRequest, - api_key: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep -) -> UnQuestResponse: - """ - Extract specific HTML attributes from webpage elements. - - Features: - - CSS selector-based extraction - - Multiple processing types (raw, cleaned, urls_resolved, numeric, boolean, list) - - Advanced filtering and validation - - Bulk extraction operations - """ - try: - logger.info("attributes_extraction_started", - url=request.url, - rules_count=len(request.rules), - client_info=client_info) - - extractor = await get_attributes_extractor() - - # Convert request rules to service format - from app.services.attributes_extraction import AttributesExtractionConfig - - rules = [] - for rule_req in request.rules: - rules.append(AttributeExtractionRule( - selector=rule_req.selector, - attribute=rule_req.attribute, - processing=AttributeProcessingType(rule_req.processing), - filter_empty=rule_req.filter_empty, - filter_duplicates=rule_req.filter_duplicates, - limit=rule_req.limit, - transform=rule_req.transform, - validation_pattern=rule_req.validation_pattern - )) - - config = AttributesExtractionConfig( - rules=rules, - base_url=request.base_url, - include_element_context=request.include_element_context, - max_elements_per_selector=request.max_elements_per_selector, - resolve_relative_urls=request.resolve_relative_urls - ) - - # Execute extraction - result = await extractor.extract_attributes(request.url, config) - - if result.success: - return UnQuestResponse( - success=True, - data={ - "url": result.url, - "extractions": [ - { - "selector": extraction.selector, - "attribute": extraction.attribute, - "values": extraction.values, - "processed_values": extraction.processed_values, - "element_count": extraction.element_count, - "results": [ - { - "element_index": res.element_index, - "raw_value": res.raw_value, - "processed_value": res.processed_value, - "element_text": res.element_text, - "element_tag": res.element_tag, - "element_classes": res.element_classes, - "element_id": res.element_id - } - for res in extraction.results - ] if request.include_element_context else [] - } - for extraction in result.extractions - ], - "total_attributes_extracted": result.total_attributes_extracted, - "total_elements_processed": result.total_elements_processed, - "processing_time_ms": result.processing_time_ms, - "metadata": result.metadata - }, - message=f"Extracted {result.total_attributes_extracted} attributes from {result.total_elements_processed} elements" - ) - else: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f"Attributes extraction failed: {result.error}" - ) - - except Exception as e: - logger.error("attributes_extraction_failed", - url=request.url, - error=str(e), - client_info=client_info) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Attributes extraction failed: {str(e)}" - ) - - -@router.post("/scrape/advanced", summary="Advanced Scraping with All Features", status_code=status.HTTP_200_OK) -async def advanced_scrape_endpoint( - request: AdvancedScrapeRequest, - api_key: ApiKeyDep, - settings: SettingsDep, - client_info: ClientInfoDep, - background_tasks: BackgroundTasks -) -> UnQuestResponse: - """ - Perform advanced scraping with all features combined. - - Combines: - - Browser actions (click, scroll, type, etc.) - - Change tracking and monitoring - - Attributes extraction - - Multi-engine scraping - """ - try: - logger.info("advanced_scraping_started", - url=request.url, - features={ - "actions": bool(request.actions), - "change_tracking": bool(request.change_tracking), - "attributes": bool(request.attributes_extraction) - }, - client_info=client_info) - - results = {"url": request.url} - - # 1. Execute browser actions if provided - if request.actions: - actions_dicts = [action.dict(exclude_none=True) for action in request.actions] - actions_result = await execute_browser_actions( - url=request.url, - actions=actions_dicts - ) - results["actions"] = { - "success": actions_result.success, - "screenshots": actions_result.screenshots, - "scrapes": actions_result.scrapes, - "total_execution_time_ms": actions_result.total_execution_time_ms - } - - # 2. Perform standard scraping - from app.services.enhanced_scraping import get_enhanced_scraping_service - scraping_service = await get_enhanced_scraping_service() - - scraping_config = request.scraping_config or ScrapingConfig() - scrape_results = await scraping_service.scrape_urls_enhanced( - [request.url], - config=scraping_config - ) - - if scrape_results: - results["scraped_content"] = scrape_results[0].dict() - - # 3. Track changes if requested - if request.change_tracking: - tracking_service = await get_change_tracking_service() - config = ChangeTrackingConfig( - tag=request.change_tracking.tag, - threshold=request.change_tracking.threshold, - compare_text=request.change_tracking.compare_text, - compare_html=request.change_tracking.compare_html, - notification_webhook=request.change_tracking.notification_webhook - ) - - tracking_result = await tracking_service.track_content_changes(request.url, config) - results["change_tracking"] = { - "change_status": tracking_result.tracking_data.change_status.value, - "change_percentage": tracking_result.tracking_data.change_percentage, - "significant_changes": tracking_result.tracking_data.significant_changes - } - - # 4. Extract attributes if requested - if request.attributes_extraction: - extractor = await get_attributes_extractor() - - from app.services.attributes_extraction import AttributesExtractionConfig - - rules = [] - for rule_req in request.attributes_extraction.rules: - rules.append(AttributeExtractionRule( - selector=rule_req.selector, - attribute=rule_req.attribute, - processing=AttributeProcessingType(rule_req.processing) - )) - - config = AttributesExtractionConfig(rules=rules) - extraction_result = await extractor.extract_attributes(request.url, config) - - results["attributes"] = { - "total_attributes_extracted": extraction_result.total_attributes_extracted, - "extractions": [ - { - "selector": ext.selector, - "attribute": ext.attribute, - "values": ext.values[:10] # Limit to first 10 values - } - for ext in extraction_result.extractions - ] - } - - return UnQuestResponse( - success=True, - data=results, - message="Advanced scraping completed successfully" - ) - - except Exception as e: - logger.error("advanced_scraping_failed", - url=request.url, - error=str(e), - client_info=client_info) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Advanced scraping failed: {str(e)}" - ) diff --git a/apps/backend/app/config.py b/apps/backend/app/config.py deleted file mode 100644 index b130e64..0000000 --- a/apps/backend/app/config.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -Application configuration management using Pydantic Settings. -""" -from typing import List, Literal, Optional -from pydantic import HttpUrl, PostgresDsn, field_validator, ValidationInfo -from pydantic_settings import BaseSettings -from pydantic_settings import SettingsConfigDict -import os - - -class Settings(BaseSettings): - """Application settings with environment variable support.""" - - # Application - app_name: str = "UnSearch API" - version: str = "1.0.0" - debug: bool = False - environment: Literal["development", "staging", "production"] = "development" - - # API Configuration - api_prefix: str = "/api/v1" - docs_url: Optional[str] = "/docs" - openapi_url: Optional[str] = "/openapi.json" - - # Server - host: str = "0.0.0.0" - port: int = 8000 - workers: int = 4 - - # SearXNG - searxng_url: HttpUrl = "http://localhost:8080" - searxng_timeout: int = 10 - searxng_max_retries: int = 3 - searxng_enabled_engines: List[str] = ["google", "bing", "duckduckgo", "startpage", "qwant"] - - # Redis - redis_url: str = "redis://localhost:6379" - redis_max_connections: int = 20 - cache_default_ttl: int = 3600 - cache_compression: bool = True - - # Upstash Redis (cloud Redis service) - upstash_redis_rest_url: Optional[str] = None - upstash_redis_rest_token: Optional[str] = None - - # Database - database_url: PostgresDsn = "postgresql://user:pass@localhost:5432/database" - database_pool_size: int = 10 - database_max_overflow: int = 20 - database_pool_timeout: int = 30 - database_echo: bool = False - - # Security - api_key_header: str = "X-API-Key" - api_keys: List[str] = [] # Load from environment - allowed_origins: List[str] = ["*"] - cors_credentials: bool = True - cors_methods: List[str] = ["GET", "POST", "PUT", "DELETE", "OPTIONS"] - cors_headers: List[str] = ["*"] - - # Rate Limiting - rate_limit_enabled: bool = True - rate_limit_default: str = "1000/hour" - rate_limit_burst: int = 100 - rate_limit_storage_url: Optional[str] = None # Uses Redis if None - - # Scraping - scraping_max_concurrent: int = 10 - scraping_timeout: int = 30 - scraping_user_agent: str = "UnSearch-API/1.0 (+https://github.com/UnSearch)" - scraping_respect_robots_txt: bool = True - scraping_min_delay_seconds: float = 0.5 - scraping_max_retries: int = 3 - scraping_javascript_enabled: bool = False - # Puppeteer JS rendering service - puppeteer_enabled: bool = True - puppeteer_service_url: Optional[HttpUrl] = "http://localhost:9223" - puppeteer_timeout: int = 30 - puppeteer_default_wait_until: str = "networkidle0" - - # Advanced Features Configuration - # Fire Engine service for advanced scraping - fire_engine_url: Optional[str] = None - fire_engine_timeout: int = 60 - - # Multi-provider search API keys - serper_api_key: Optional[str] = None - searchapi_key: Optional[str] = None - - # LLM Configuration - openai_api_key: Optional[str] = None - openai_model: str = "gpt-4" - openai_max_tokens: int = 4096 - - # Batch Operations - batch_max_concurrent_jobs: int = 5 - batch_max_workers: int = 10 - batch_job_timeout: int = 3600 # 1 hour - - # Playwright service for advanced rendering - playwright_service_url: Optional[str] = None - playwright_timeout: int = 60 - - # Content Processing - content_max_size_mb: int = 10 - content_min_text_length: int = 50 - content_language_detection: bool = True - content_quality_threshold: float = 0.3 - - # Monitoring - enable_metrics: bool = True - metrics_path: str = "/metrics" - log_level: str = "INFO" - log_format: str = "json" - log_file: Optional[str] = None - - # JWT Settings - secret_key: str = "your-secret-key-change-this-in-production" - jwt_algorithm: str = "HS256" - access_token_expire_minutes: int = 1440 # 24 hours - - # Stripe Settings - stripe_secret_key: str = "" - stripe_publishable_key: str = "" - stripe_webhook_secret: str = "" - stripe_price_id_pro: str = "" # Will be set after creating products - - # Celery - celery_broker_url: Optional[str] = None # Uses Redis if None - celery_result_backend: Optional[str] = None # Uses Redis if None - celery_task_time_limit: int = 300 - celery_task_soft_time_limit: int = 240 - celery_worker_concurrency: int = 4 - - @field_validator("api_keys", mode="before") - def parse_api_keys(cls, v): - """Parse API keys from comma-separated string.""" - if isinstance(v, str): - return [key.strip() for key in v.split(",") if key.strip()] - return v or [] - - @field_validator("searxng_enabled_engines", "allowed_origins", "cors_methods", "cors_headers", mode="before") - def parse_list_fields(cls, v): - """Parse list fields from comma-separated strings.""" - if isinstance(v, str): - if not v.strip(): # Handle empty strings - return [] - return [item.strip() for item in v.split(",") if item.strip()] - return v or [] - - @field_validator("celery_broker_url", mode="before") - def set_celery_broker(cls, v, info: ValidationInfo): - """Use Redis URL for Celery broker if not specified.""" - if v: - return v - # Get redis_url from the data being validated - data = info.data if info.data else {} - return data.get("redis_url", "redis://localhost:6379") - - @field_validator("celery_result_backend", mode="before") - def set_celery_backend(cls, v, info: ValidationInfo): - """Use Redis URL for Celery backend if not specified.""" - if v: - return v - # Get redis_url from the data being validated - data = info.data if info.data else {} - return data.get("redis_url", "redis://localhost:6379") - - @field_validator("rate_limit_storage_url", mode="before") - def set_rate_limit_storage(cls, v, info: ValidationInfo): - """Use Redis URL for rate limit storage if not specified.""" - if v: - return v - # Get redis_url from the data being validated - data = info.data if info.data else {} - return data.get("redis_url", "redis://localhost:6379") - - @field_validator("database_url", mode="before") - def validate_database_url(cls, v): - """Ensure database URL is properly formatted.""" - if isinstance(v, str) and not v.startswith(("postgresql://", "postgres://")): - raise ValueError("Database URL must be a valid PostgreSQL connection string") - return v - - model_config = SettingsConfigDict( - env_file=".env", - env_file_encoding="utf-8", - case_sensitive=False, - extra="ignore", - env_parse_none_str="None" # Handle None values properly - ) - - -# Global settings instance -settings = Settings() - - -def get_settings() -> Settings: - """Get application settings.""" - return settings diff --git a/apps/backend/app/main.py b/apps/backend/app/main.py deleted file mode 100644 index 497b351..0000000 --- a/apps/backend/app/main.py +++ /dev/null @@ -1,323 +0,0 @@ -""" -Main FastAPI application entry point. -""" -import time -from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, status -from fastapi.middleware.cors import CORSMiddleware -from fastapi.middleware.gzip import GZipMiddleware -from fastapi.middleware.trustedhost import TrustedHostMiddleware -from fastapi.responses import JSONResponse, Response -from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST -from slowapi import Limiter, _rate_limit_exceeded_handler -from slowapi.util import get_remote_address -from slowapi.errors import RateLimitExceeded -from slowapi.middleware import SlowAPIMiddleware -import structlog -from structlog.contextvars import bind_contextvars, clear_contextvars - -from app.config import get_settings -from app.api.v1 import search, auth, billing -from app.api.v1 import enhanced_search -from app.api.v2 import advanced_endpoints -from app.models.responses import ErrorResponse -from app.services.database import get_database_service -from app.services.searxng import get_searxng_service -from app.services.scraping import get_scraping_service -from app.services.cache import get_cache_service -from app.utils.error_handlers import register_exception_handlers -from app.utils.security import SecurityHeaders - -# Configure structured logging -structlog.configure( - processors=[ - structlog.contextvars.merge_contextvars, - structlog.processors.add_log_level, - structlog.processors.TimeStamper(fmt="iso"), - structlog.dev.ConsoleRenderer() if get_settings().debug else structlog.processors.JSONRenderer() - ], - context_class=dict, - logger_factory=structlog.PrintLoggerFactory(), - wrapper_class=structlog.make_filtering_bound_logger(get_settings().log_level), -) - -logger = structlog.get_logger() -settings = get_settings() - -# Prometheus metrics -REQUEST_COUNT = Counter( - 'http_requests_total', - 'Total HTTP requests', - ['method', 'endpoint', 'status'] -) -REQUEST_DURATION = Histogram( - 'http_request_duration_seconds', - 'HTTP request duration', - ['method', 'endpoint'] -) -SEARCH_REQUESTS = Counter( - 'search_requests_total', - 'Total search requests', - ['engine', 'cached'] -) -SCRAPING_REQUESTS = Counter( - 'scraping_requests_total', - 'Total scraping requests', - ['success'] -) - - -@asynccontextmanager -async def lifespan(app: FastAPI): - """ - Application lifespan manager. - - Handles startup and shutdown events. - """ - # Startup - logger.info("application_startup", version=settings.version, environment=settings.environment) - - # Initialize services - db_service = await get_database_service() - searxng_service = await get_searxng_service() - scraping_service = await get_scraping_service() - cache_service = await get_cache_service() - - # Store services in app state for cleanup - app.state.db = db_service - app.state.searxng = searxng_service - app.state.scraper = scraping_service - app.state.cache = cache_service - - # Store startup time for uptime calculation - app.state.startup_time = time.time() - - logger.info("services_initialized") - - yield - - # Shutdown - logger.info("application_shutdown") - - # Cleanup services - await app.state.searxng.close() - await app.state.scraper.close() - await app.state.cache.close() - await app.state.db.close() - - logger.info("services_closed") - - -# Create FastAPI app -app = FastAPI( - title=settings.app_name, - version=settings.version, - docs_url=settings.docs_url, - openapi_url=settings.openapi_url, - lifespan=lifespan, - description="Privacy-respecting web search and scraping API powered by SearXNG and BeautifulSoup4", - contact={ - "name": "UnSearch API", - "url": "https://github.com/Rakesh1002/unsearch", - "email": "support@unsearch.dev" - }, - license_info={ - "name": "AGPL-3.0", - "url": "https://www.gnu.org/licenses/agpl-3.0.html" - } -) - -# Configure rate limiting -limiter = Limiter( - key_func=get_remote_address, - default_limits=[settings.rate_limit_default], - enabled=settings.rate_limit_enabled, - storage_uri=settings.rate_limit_storage_url -) -app.state.limiter = limiter -app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) -app.add_middleware(SlowAPIMiddleware) - -# Register exception handlers -register_exception_handlers(app) - -# Add middlewares -app.add_middleware( - CORSMiddleware, - allow_origins=settings.allowed_origins, - allow_credentials=settings.cors_credentials, - allow_methods=settings.cors_methods, - allow_headers=settings.cors_headers, - expose_headers=["X-Request-ID", "X-RateLimit-Limit", "X-RateLimit-Remaining"] -) - -app.add_middleware(GZipMiddleware, minimum_size=1000) - -if settings.environment == "production": - app.add_middleware( - TrustedHostMiddleware, - allowed_hosts=["*.unsearch.dev", "localhost"] - ) - - -@app.middleware("http") -async def logging_middleware(request: Request, call_next): - """ - Middleware for request logging and metrics. - """ - # Generate request ID - request_id = request.headers.get("X-Request-ID", str(time.time())) - - # Bind request context for structured logging - bind_contextvars( - request_id=request_id, - path=request.url.path, - method=request.method, - client_ip=request.client.host if request.client else None - ) - - # Track request duration - start_time = time.time() - - try: - response = await call_next(request) - - # Calculate duration - duration = time.time() - start_time - - # Update metrics - REQUEST_COUNT.labels( - method=request.method, - endpoint=request.url.path, - status=response.status_code - ).inc() - - REQUEST_DURATION.labels( - method=request.method, - endpoint=request.url.path - ).observe(duration) - - # Add response headers - response.headers["X-Request-ID"] = request_id - response.headers["X-Response-Time"] = f"{duration:.3f}" - - # Add security headers - security_headers = SecurityHeaders.get_headers() - for header, value in security_headers.items(): - response.headers[header] = value - - # Log request - logger.info( - "http_request", - status_code=response.status_code, - duration_seconds=duration - ) - - return response - - except Exception as e: - duration = time.time() - start_time - - logger.error( - "http_request_error", - error=str(e), - duration_seconds=duration - ) - - raise - - finally: - clear_contextvars() - - -@app.exception_handler(Exception) -async def global_exception_handler(request: Request, exc: Exception): - """ - Global exception handler for unhandled errors. - """ - logger.error( - "unhandled_exception", - error=str(exc), - error_type=type(exc).__name__, - path=request.url.path - ) - - error_response = ErrorResponse( - error="InternalServerError", - message="An unexpected error occurred", - request_id=request.headers.get("X-Request-ID"), - details={"error": str(exc)} if settings.debug else None - ) - - return JSONResponse( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content=error_response.dict() - ) - - -# Include routers -app.include_router( - search.router, - prefix=settings.api_prefix -) - -app.include_router( - auth.router, - prefix=settings.api_prefix -) - -app.include_router( - billing.router, - prefix=settings.api_prefix -) - -# Enhanced search endpoints (v1) -app.include_router( - enhanced_search.router, - prefix=settings.api_prefix -) - -# Advanced endpoints (v2) -app.include_router( - advanced_endpoints.router, - prefix=settings.api_prefix -) - - -@app.get("/", include_in_schema=False) -async def root(): - """Root endpoint.""" - return { - "name": settings.app_name, - "version": settings.version, - "docs": settings.docs_url, - "health": "/health" - } - - -@app.get("/health", include_in_schema=False) -async def health(): - """Basic health check endpoint.""" - return {"status": "healthy"} - - -@app.get("/metrics", include_in_schema=False) -async def metrics(): - """Prometheus metrics endpoint.""" - if not settings.enable_metrics: - return {"error": "Metrics disabled"} - - return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) - - -if __name__ == "__main__": - import uvicorn - - uvicorn.run( - "app.main:app", - host=settings.host, - port=settings.port, - workers=settings.workers, - log_level=settings.log_level.lower(), - reload=settings.debug - ) diff --git a/apps/backend/app/middleware/rate_limit.py b/apps/backend/app/middleware/rate_limit.py deleted file mode 100644 index 5c946a4..0000000 --- a/apps/backend/app/middleware/rate_limit.py +++ /dev/null @@ -1,276 +0,0 @@ -""" -Plan-based rate limiting middleware. -""" -from typing import Optional, Tuple -from datetime import datetime, timedelta -from fastapi import Request, HTTPException, status -from slowapi import Limiter -from slowapi.util import get_remote_address -import structlog -import redis.asyncio as redis - -from app.config import get_settings -from app.services.auth_service import AuthService -from app.models.users import User, PlanType - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class PlanBasedRateLimiter: - """Rate limiter that considers user subscription plans.""" - - def __init__(self): - self.redis_client = None - self.default_limits = { - PlanType.FREE: "100/hour", - PlanType.PRO: "1000/hour", - PlanType.ENTERPRISE: "10000/hour" - } - - async def initialize(self): - """Initialize Redis connection.""" - if not self.redis_client: - self.redis_client = redis.from_url( - settings.redis_url, - decode_responses=True - ) - - async def close(self): - """Close Redis connection.""" - if self.redis_client: - await self.redis_client.close() - - def _parse_rate_limit(self, limit_str: str) -> Tuple[int, int]: - """Parse rate limit string (e.g., '100/hour') to count and seconds.""" - parts = limit_str.split('/') - if len(parts) != 2: - return 100, 3600 # Default - - count = int(parts[0]) - - period_map = { - 'second': 1, - 'minute': 60, - 'hour': 3600, - 'day': 86400 - } - - period = parts[1].lower() - seconds = period_map.get(period, 3600) - - return count, seconds - - async def check_rate_limit( - self, - request: Request, - user: Optional[User] = None - ) -> bool: - """Check if request exceeds rate limit.""" - await self.initialize() - - # Determine rate limit based on user plan - if user: - # Get user's subscription - subscription = user.current_subscription - if subscription and subscription.rate_limit: - limit_str = subscription.rate_limit - else: - limit_str = self.default_limits.get(user.current_plan, "100/hour") - - key = f"rate_limit:user:{user.id}" - else: - # Anonymous users get minimal rate limit - limit_str = "10/hour" - client_ip = get_remote_address(request) - key = f"rate_limit:ip:{client_ip}" - - # Parse limit - max_requests, period_seconds = self._parse_rate_limit(limit_str) - - # Check current count - current = await self.redis_client.get(key) - current_count = int(current) if current else 0 - - if current_count >= max_requests: - return False - - # Increment counter - pipe = self.redis_client.pipeline() - pipe.incr(key) - pipe.expire(key, period_seconds) - await pipe.execute() - - # Set rate limit headers - request.state.rate_limit_limit = max_requests - request.state.rate_limit_remaining = max_requests - current_count - 1 - request.state.rate_limit_reset = datetime.utcnow() + timedelta(seconds=period_seconds) - - return True - - async def check_usage_limit( - self, - user: User, - search: bool = False, - scrape: bool = False - ) -> Tuple[bool, Optional[str]]: - """Check if user has exceeded monthly usage limits.""" - # Get current month usage - now = datetime.utcnow() - month_key = f"usage:{user.id}:{now.year}:{now.month}" - - search_key = f"{month_key}:search" - scrape_key = f"{month_key}:scrape" - - # Get current counts - search_count = await self.redis_client.get(search_key) or 0 - scrape_count = await self.redis_client.get(scrape_key) or 0 - - search_count = int(search_count) - scrape_count = int(scrape_count) - - # Get limits from subscription - subscription = user.current_subscription - - if subscription: - search_limit = subscription.search_limit - scrape_limit = subscription.scrape_limit - else: - # Free plan defaults - search_limit = 1000 - scrape_limit = 10000 - - # Check limits - if search: - if search_limit and search_count >= search_limit: - return False, f"Monthly search limit ({search_limit}) exceeded" - - if scrape: - if scrape_limit and scrape_count >= scrape_limit: - return False, f"Monthly scrape limit ({scrape_limit}) exceeded" - - return True, None - - async def increment_usage( - self, - user: User, - search_count: int = 0, - scrape_count: int = 0 - ): - """Increment usage counters.""" - now = datetime.utcnow() - month_key = f"usage:{user.id}:{now.year}:{now.month}" - - pipe = self.redis_client.pipeline() - - if search_count > 0: - search_key = f"{month_key}:search" - pipe.incrby(search_key, search_count) - pipe.expire(search_key, 35 * 24 * 3600) # Expire after 35 days - - if scrape_count > 0: - scrape_key = f"{month_key}:scrape" - pipe.incrby(scrape_key, scrape_count) - pipe.expire(scrape_key, 35 * 24 * 3600) - - await pipe.execute() - - async def get_usage_stats(self, user: User) -> dict: - """Get current usage statistics.""" - now = datetime.utcnow() - month_key = f"usage:{user.id}:{now.year}:{now.month}" - - search_key = f"{month_key}:search" - scrape_key = f"{month_key}:scrape" - - search_count = await self.redis_client.get(search_key) or 0 - scrape_count = await self.redis_client.get(scrape_key) or 0 - - subscription = user.current_subscription - - if subscription: - search_limit = subscription.search_limit - scrape_limit = subscription.scrape_limit - else: - search_limit = 1000 - scrape_limit = 10000 - - return { - "searches": { - "used": int(search_count), - "limit": search_limit, - "remaining": (search_limit - int(search_count)) if search_limit else None, - "unlimited": search_limit is None - }, - "scrapes": { - "used": int(scrape_count), - "limit": scrape_limit, - "remaining": (scrape_limit - int(scrape_count)) if scrape_limit else None, - "unlimited": scrape_limit is None - } - } - - -# Singleton instance -_rate_limiter: Optional[PlanBasedRateLimiter] = None - - -async def get_rate_limiter() -> PlanBasedRateLimiter: - """Get or create rate limiter instance.""" - global _rate_limiter - - if _rate_limiter is None: - _rate_limiter = PlanBasedRateLimiter() - await _rate_limiter.initialize() - - return _rate_limiter - - -async def rate_limit_middleware(request: Request, call_next): - """Middleware to enforce rate limits based on user plan.""" - # Skip rate limiting for certain paths - skip_paths = ["/health", "/metrics", "/docs", "/openapi.json", "/favicon.ico"] - if request.url.path in skip_paths: - return await call_next(request) - - # Skip webhooks - if "/webhook/" in request.url.path: - return await call_next(request) - - # Get user from request if authenticated - user = None - if hasattr(request.state, "user"): - user = request.state.user - - # Check rate limit - rate_limiter = await get_rate_limiter() - allowed = await rate_limiter.check_rate_limit(request, user) - - if not allowed: - # Get limit info for error message - if user: - subscription = user.current_subscription - limit_str = subscription.rate_limit if subscription else "100/hour" - else: - limit_str = "10/hour" - - raise HTTPException( - status_code=status.HTTP_429_TOO_MANY_REQUESTS, - detail=f"Rate limit exceeded. Limit: {limit_str}", - headers={ - "X-RateLimit-Limit": str(request.state.rate_limit_limit), - "X-RateLimit-Remaining": "0", - "X-RateLimit-Reset": request.state.rate_limit_reset.isoformat(), - "Retry-After": str(int((request.state.rate_limit_reset - datetime.utcnow()).total_seconds())) - } - ) - - # Add rate limit headers to response - response = await call_next(request) - - if hasattr(request.state, "rate_limit_limit"): - response.headers["X-RateLimit-Limit"] = str(request.state.rate_limit_limit) - response.headers["X-RateLimit-Remaining"] = str(request.state.rate_limit_remaining) - response.headers["X-RateLimit-Reset"] = request.state.rate_limit_reset.isoformat() - - return response diff --git a/apps/backend/app/models/__init__.py b/apps/backend/app/models/__init__.py deleted file mode 100644 index 18a6ccd..0000000 --- a/apps/backend/app/models/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Data models for the UnSearch API. -""" -from app.models.requests import UnQuestRequest, BatchSearchRequest, ScrapingConfig -from app.models.responses import ( - UnQuestResponse, - SearchResult, - ScrapedContent, - SearchMetadata, - ContentMetadata, - AsyncTaskResponse, - BatchSearchResponse, - ErrorResponse, - HealthResponse, - ServiceHealth, - EngineInfo, - EnginesListResponse -) - -__all__ = [ - # Request models - "UnQuestRequest", - "BatchSearchRequest", - "ScrapingConfig", - - # Response models - "UnQuestResponse", - "SearchResult", - "ScrapedContent", - "SearchMetadata", - "ContentMetadata", - "AsyncTaskResponse", - "BatchSearchResponse", - "ErrorResponse", - "HealthResponse", - "ServiceHealth", - "EngineInfo", - "EnginesListResponse" -] diff --git a/apps/backend/app/models/auth_models.py b/apps/backend/app/models/auth_models.py deleted file mode 100644 index 0b2c09f..0000000 --- a/apps/backend/app/models/auth_models.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -Pydantic models for authentication and billing requests/responses. -""" -from typing import Optional, List, Dict, Any -from datetime import datetime -from pydantic import BaseModel, EmailStr, Field, validator - - -# ==================== Request Models ==================== - -class UserRegisterRequest(BaseModel): - """User registration request.""" - email: EmailStr - password: str = Field(..., min_length=8, max_length=100) - full_name: Optional[str] = Field(None, max_length=255) - company: Optional[str] = Field(None, max_length=255) - - @validator('password') - def validate_password(cls, v): - """Ensure password meets complexity requirements.""" - if not any(char.isdigit() for char in v): - raise ValueError('Password must contain at least one digit') - if not any(char.isupper() for char in v): - raise ValueError('Password must contain at least one uppercase letter') - if not any(char.islower() for char in v): - raise ValueError('Password must contain at least one lowercase letter') - return v - - -class UserLoginRequest(BaseModel): - """User login request.""" - email: EmailStr - password: str - - -class RefreshTokenRequest(BaseModel): - """Refresh token request.""" - refresh_token: str - - -class CreateAPIKeyRequest(BaseModel): - """Create API key request.""" - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - scopes: Optional[List[str]] = ["read", "write"] - - -class ResetPasswordRequest(BaseModel): - """Password reset request.""" - email: EmailStr - - -class ChangePasswordRequest(BaseModel): - """Change password request.""" - current_password: str - new_password: str = Field(..., min_length=8, max_length=100) - - -class CreateSubscriptionRequest(BaseModel): - """Create subscription request.""" - price_id: str - trial_days: Optional[int] = 0 - - -class UpdateSubscriptionRequest(BaseModel): - """Update subscription request.""" - price_id: str - - -class CreateCheckoutSessionRequest(BaseModel): - """Create Stripe checkout session request.""" - price_id: str - success_url: str - cancel_url: str - trial_days: Optional[int] = 0 - - -# ==================== Response Models ==================== - -class UserResponse(BaseModel): - """User response model.""" - id: int - email: str - full_name: Optional[str] - company: Optional[str] - is_verified: bool - plan: str - created_at: datetime - - -class LoginResponse(BaseModel): - """Login response model.""" - access_token: str - refresh_token: str - token_type: str = "bearer" - expires_in: int - user: Optional[Dict[str, Any]] = None - - -class APIKeyResponse(BaseModel): - """API key response model.""" - id: int - key: str - name: str - description: Optional[str] - scopes: List[str] - last_used_at: Optional[datetime] - created_at: datetime - - -class UsageResponse(BaseModel): - """Usage statistics response.""" - period: Dict[str, str] - searches: Dict[str, Any] - scrapes: Dict[str, Any] - api_calls: int - usage_by_engine: Dict[str, int] - usage_by_day: Dict[str, int] - - -class SubscriptionResponse(BaseModel): - """Subscription response model.""" - id: int - plan_type: str - status: str - amount: float - currency: str - interval: str - search_limit: Optional[int] - scrape_limit: Optional[int] - rate_limit: str - features: Dict[str, bool] - current_period_start: datetime - current_period_end: datetime - trial_end: Optional[datetime] - cancelled_at: Optional[datetime] - is_active: bool - days_remaining: Optional[int] = None - - -class PlanResponse(BaseModel): - """Subscription plan response.""" - id: int - name: str - display_name: str - description: Optional[str] - price: float - currency: str - interval: str - search_limit: Optional[int] - scrape_limit: Optional[int] - rate_limit: str - features: Dict[str, bool] - - -class InvoiceResponse(BaseModel): - """Invoice response model.""" - id: int - invoice_number: Optional[str] - status: str - amount_due: float - amount_paid: float - currency: str - period_start: Optional[datetime] - period_end: Optional[datetime] - paid_at: Optional[datetime] - invoice_pdf: Optional[str] - hosted_invoice_url: Optional[str] - created_at: datetime - - -class CheckoutSessionResponse(BaseModel): - """Checkout session response.""" - checkout_url: str - - -class BillingPortalResponse(BaseModel): - """Billing portal response.""" - portal_url: str - - -class OAuthSyncRequest(BaseModel): - """OAuth sync request for third-party providers (e.g., GitHub).""" - provider: str - oauth_id: str - email: EmailStr - full_name: Optional[str] = None - avatar_url: Optional[str] = None diff --git a/apps/backend/app/models/database.py b/apps/backend/app/models/database.py deleted file mode 100644 index 33c0c13..0000000 --- a/apps/backend/app/models/database.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -SQLAlchemy database models. -""" -from sqlalchemy import Column, Integer, String, DateTime, Float, Boolean, JSON, Text, Index, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship -from sqlalchemy.sql import func -from datetime import datetime -import uuid - -Base = declarative_base() - - -class APIKey(Base): - """API key management.""" - __tablename__ = "api_keys" - - id = Column(Integer, primary_key=True) - key = Column(String(64), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=False) - description = Column(Text) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_used_at = Column(DateTime(timezone=True)) - is_active = Column(Boolean, default=True) - rate_limit_override = Column(String(50)) # e.g., "5000/hour" - extra_metadata = Column(JSON, default={}) - - # Relationships - requests = relationship("SearchRequest", back_populates="api_key") - - __table_args__ = ( - Index("idx_api_keys_active", "is_active"), - ) - - -class SearchRequest(Base): - """Search request logging and analytics.""" - __tablename__ = "search_requests" - - id = Column(Integer, primary_key=True) - request_id = Column(String(36), unique=True, nullable=False, default=lambda: str(uuid.uuid4())) - api_key_id = Column(Integer, ForeignKey("api_keys.id")) - query = Column(Text, nullable=False) - engines = Column(JSON, nullable=False) - max_results = Column(Integer, nullable=False) - language = Column(String(2)) - safe_search = Column(String(10)) - - # Performance metrics - search_time_ms = Column(Integer) - scraping_time_ms = Column(Integer) - total_time_ms = Column(Integer) - results_count = Column(Integer) - scraped_count = Column(Integer) - - # Cache info - cache_hit = Column(Boolean, default=False) - cache_key = Column(String(64)) - - # Request metadata - client_ip = Column(String(45)) # IPv6 support - user_agent = Column(Text) - request_headers = Column(JSON) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - completed_at = Column(DateTime(timezone=True)) - - # Relationships - api_key = relationship("APIKey", back_populates="requests") - results = relationship("SearchResult", back_populates="request", cascade="all, delete-orphan") - - __table_args__ = ( - Index("idx_search_requests_created", "created_at"), - Index("idx_search_requests_query", "query"), - Index("idx_search_requests_cache_key", "cache_key"), - ) - - -class SearchResult(Base): - """Individual search result storage.""" - __tablename__ = "search_results" - - id = Column(Integer, primary_key=True) - request_id = Column(Integer, ForeignKey("search_requests.id"), nullable=False) - rank = Column(Integer, nullable=False) - title = Column(Text, nullable=False) - url = Column(Text, nullable=False) - snippet = Column(Text) - engine = Column(String(50), nullable=False) - score = Column(Float) - - # Scraping results - scraped_successfully = Column(Boolean, default=False) - scraped_content = Column(JSON) # Compressed JSON of ScrapedContent - scraping_error = Column(Text) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - request = relationship("SearchRequest", back_populates="results") - - __table_args__ = ( - Index("idx_search_results_request", "request_id"), - Index("idx_search_results_url", "url"), - ) - - -class ScrapingJob(Base): - """Async scraping job tracking.""" - __tablename__ = "scraping_jobs" - - id = Column(Integer, primary_key=True) - job_id = Column(String(36), unique=True, nullable=False, default=lambda: str(uuid.uuid4())) - task_id = Column(String(255)) # Celery task ID - urls = Column(JSON, nullable=False) - config = Column(JSON, nullable=False) - status = Column(String(20), nullable=False, default="pending") # pending, processing, completed, failed - - # Results - results = Column(JSON) - error_message = Column(Text) - - # Webhook - webhook_url = Column(Text) - webhook_attempts = Column(Integer, default=0) - webhook_last_attempt = Column(DateTime(timezone=True)) - webhook_success = Column(Boolean) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - started_at = Column(DateTime(timezone=True)) - completed_at = Column(DateTime(timezone=True)) - - __table_args__ = ( - Index("idx_scraping_jobs_status", "status"), - Index("idx_scraping_jobs_created", "created_at"), - ) - - -class CacheEntry(Base): - """Cache metadata for analytics.""" - __tablename__ = "cache_entries" - - id = Column(Integer, primary_key=True) - cache_key = Column(String(64), unique=True, nullable=False) - query_hash = Column(String(64), nullable=False) - size_bytes = Column(Integer) - hit_count = Column(Integer, default=0) - ttl_seconds = Column(Integer) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_accessed_at = Column(DateTime(timezone=True)) - expires_at = Column(DateTime(timezone=True)) - - __table_args__ = ( - Index("idx_cache_entries_key", "cache_key"), - Index("idx_cache_entries_expires", "expires_at"), - ) - - -class ErrorLog(Base): - """Error logging for debugging.""" - __tablename__ = "error_logs" - - id = Column(Integer, primary_key=True) - request_id = Column(String(36)) - error_type = Column(String(100), nullable=False) - error_message = Column(Text, nullable=False) - error_details = Column(JSON) - stack_trace = Column(Text) - - # Context - endpoint = Column(String(255)) - method = Column(String(10)) - status_code = Column(Integer) - client_ip = Column(String(45)) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - __table_args__ = ( - Index("idx_error_logs_created", "created_at"), - Index("idx_error_logs_type", "error_type"), - Index("idx_error_logs_request", "request_id"), - ) diff --git a/apps/backend/app/models/requests.py b/apps/backend/app/models/requests.py deleted file mode 100644 index 34ef238..0000000 --- a/apps/backend/app/models/requests.py +++ /dev/null @@ -1,479 +0,0 @@ -""" -API request models using Pydantic v2. -""" -from typing import Dict, List, Literal, Optional, Union, Any -from pydantic import BaseModel, Field, HttpUrl, validator -import re - - -class UnQuestRequest(BaseModel): - """Main search and scrape request model.""" - - query: str = Field( - ..., - min_length=1, - max_length=500, - description="Search query", - examples=["Python web scraping tutorial"] - ) - - engines: List[str] = Field( - default=["google", "bing", "duckduckgo"], - description="Search engines to use", - min_items=1, - max_items=10 - ) - - max_results: int = Field( - default=10, - ge=1, - le=100, - description="Maximum results to return" - ) - - scrape_content: bool = Field( - default=True, - description="Whether to scrape page content" - ) - - scrape_selectors: Optional[Dict[str, str]] = Field( - default=None, - description="Custom CSS selectors for content extraction", - examples=[{"title": "h1", "content": "article", "author": ".author-name"}] - ) - - output_format: Literal["json", "markdown"] = Field( - default="json", - description="Response format (json or markdown)" - ) - - cache_ttl: int = Field( - default=3600, - ge=0, - le=86400, - description="Cache TTL in seconds (0 to disable caching)" - ) - - language: str = Field( - default="en", - pattern="^[a-z]{2}$", - description="Language code (ISO 639-1)" - ) - - safe_search: Literal["strict", "moderate", "off"] = Field( - default="moderate", - description="Safe search level" - ) - - include_images: bool = Field( - default=True, - description="Extract images from scraped content" - ) - - include_links: bool = Field( - default=True, - description="Extract links from scraped content" - ) - - timeout: int = Field( - default=30, - ge=5, - le=120, - description="Request timeout in seconds" - ) - - async_mode: bool = Field( - default=False, - description="Process request asynchronously" - ) - - # Advanced crawling options - js_mode: bool = Field( - default=False, - description="Use headless browser (Puppeteer) for JavaScript rendering" - ) - screenshot: bool = Field( - default=False, - description="Capture screenshot when js_mode is enabled" - ) - pdf: bool = Field( - default=False, - description="Capture PDF when js_mode is enabled" - ) - - webhook_url: Optional[HttpUrl] = Field( - default=None, - description="Webhook URL for async results" - ) - - @validator("query") - def sanitize_query(cls, v): - """Sanitize search query to prevent injection.""" - # Remove any potential script tags or SQL injection attempts - v = re.sub(r'<[^>]*>', '', v) - v = re.sub(r'[;\'"\\]', '', v) - return v.strip() - - @validator("engines") - def validate_engines(cls, v): - """Validate search engines.""" - allowed_engines = { - "google", "bing", "duckduckgo", "startpage", - "qwant", "yahoo", "searx", "brave", "ecosia" - } - invalid = set(v) - allowed_engines - if invalid: - raise ValueError(f"Invalid engines: {invalid}") - return list(set(v)) # Remove duplicates - - @validator("webhook_url") - def validate_webhook_url(cls, v, values): - """Validate webhook URL is required for async mode.""" - if values.get("async_mode") and not v: - raise ValueError("webhook_url is required when async_mode is True") - return v - - -class BatchSearchRequest(BaseModel): - """Batch search request for multiple queries.""" - - queries: List[str] = Field( - ..., - min_items=1, - max_items=100, - description="List of search queries" - ) - - engines: List[str] = Field( - default=["google", "bing"], - description="Search engines to use for all queries" - ) - - max_results_per_query: int = Field( - default=5, - ge=1, - le=20, - description="Maximum results per query" - ) - - scrape_content: bool = Field( - default=False, - description="Whether to scrape content (disabled by default for batch)" - ) - - parallel_requests: int = Field( - default=5, - ge=1, - le=20, - description="Number of parallel requests" - ) - - -class ScrapingConfig(BaseModel): - """Configuration for content scraping.""" - - urls: List[HttpUrl] = Field( - ..., - min_items=1, - max_items=50, - description="URLs to scrape" - ) - - selectors: Optional[Dict[str, str]] = Field( - default=None, - description="CSS selectors for extraction" - ) - - extract_text: bool = Field(default=True) - extract_images: bool = Field(default=True) - extract_links: bool = Field(default=True) - extract_metadata: bool = Field(default=True) - - javascript_rendering: bool = Field( - default=False, - description="Enable JavaScript rendering (slower)" - ) - js_mode: bool = Field( - default=False, - description="Alias for javascript_rendering" - ) - - wait_time: int = Field( - default=0, - ge=0, - le=10, - description="Wait time in seconds after page load" - ) - - headers: Optional[Dict[str, str]] = Field( - default=None, - description="Custom HTTP headers" - ) - - cookies: Optional[Dict[str, str]] = Field( - default=None, - description="Custom cookies" - ) - - # Output formatting - response_format: Literal["json", "markdown"] = Field( - default="json", - description="Return JSON or Markdown content" - ) - - # Browser / identity / media options (used in js_mode) - screenshot: bool = Field(default=False, description="Capture screenshot in js_mode") - pdf: bool = Field(default=False, description="Capture PDF in js_mode") - include_html: bool = Field(default=False, description="Include raw HTML in response") - user_agent: Optional[str] = Field(default=None, description="Override User-Agent") - proxy: Optional[str] = Field(default=None, description="HTTP proxy in host:port or scheme://host:port") - wait_until: Optional[Literal["load", "domcontentloaded", "networkidle0", "networkidle2"]] = Field( - default=None, - description="Puppeteer navigation waitUntil option" - ) - - # Caching - cache_mode: Literal["enabled", "read_only", "write_only", "bypass", "disabled"] = Field( - default="enabled", - description="Crawl cache behavior" - ) - cache_ttl: int = Field( - default=86400, - ge=0, - le=7 * 24 * 3600, - description="TTL for cached HTML (seconds)" - ) - - # Dispatch/rate limiting - per_host_concurrency: int = Field( - default=2, - ge=1, - le=10, - description="Max parallel requests per host" - ) - hits_per_sec: float = Field( - default=0.0, - ge=0.0, - le=20.0, - description="Per-host request rate (0 disables rate limiting)" - ) - - # Link head extraction / scoring - link_head: bool = Field( - default=False, - description="Fetch link head/title and basic metadata" - ) - link_enrichment_concurrency: int = Field( - default=8, - ge=1, - le=32, - description="Max parallel link enrichment requests" - ) - link_timeout: int = Field( - default=5, - ge=1, - le=30, - description="Timeout (s) per link enrichment request" - ) - link_max: int = Field( - default=100, - ge=1, - le=500, - description="Max links to process for enrichment" - ) - link_score_query: Optional[str] = Field( - default=None, - description="Query to score links against (simple relevance)" - ) - link_score_threshold: Optional[float] = Field( - default=None, - ge=0.0, - le=1.0, - description="Keep links with score >= threshold" - ) - - # Advanced crawl4ai-inspired features - extraction_strategy: Literal["none", "cosine", "json_css", "regex", "llm"] = Field( - default="none", - description="Content extraction strategy" - ) - - extraction_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Configuration for the selected extraction strategy" - ) - - content_filter: Literal["none", "pruning", "bm25", "llm"] = Field( - default="none", - description="Content filtering strategy" - ) - - content_filter_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Configuration for the content filter" - ) - - markdown_generation: bool = Field( - default=False, - description="Enable enhanced markdown generation with citations" - ) - - markdown_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Configuration for markdown generation" - ) - - adaptive_crawling: bool = Field( - default=False, - description="Enable adaptive crawling with learning" - ) - - adaptive_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Configuration for adaptive crawling" - ) - - virtual_scrolling: bool = Field( - default=False, - description="Enable virtual scrolling for infinite pages" - ) - - virtual_scroll_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Configuration for virtual scrolling" - ) - - link_analysis: bool = Field( - default=False, - description="Enable intelligent link analysis and scoring" - ) - - link_analysis_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Configuration for link analysis" - ) - - -# New enhanced configuration models for crawl4ai features - -class ExtractionStrategyConfig(BaseModel): - """Configuration for extraction strategies.""" - - strategy_type: Literal["none", "cosine", "json_css", "regex", "llm"] - - # Cosine strategy options - semantic_filter: Optional[str] = None - word_count_threshold: int = 10 - max_dist: float = 0.2 - linkage_method: str = "ward" - top_k: int = 3 - model_name: str = "sentence-transformers/all-MiniLM-L6-v2" - sim_threshold: float = 0.3 - - # JSON CSS strategy options - schema: Optional[Dict[str, Any]] = None - - # Regex strategy options - patterns: Optional[Dict[str, str]] = None - - # LLM strategy options - llm_config: Optional[Dict[str, Any]] = None - extraction_type: str = "schema" - instruction: Optional[str] = None - - # Common options - verbose: bool = False - - -class ContentFilterConfig(BaseModel): - """Configuration for content filters.""" - - filter_type: Literal["none", "pruning", "bm25", "llm"] - user_query: Optional[str] = None - verbose: bool = False - - # Pruning filter options - threshold: float = 0.48 - threshold_type: str = "fixed" - min_word_threshold: int = 0 - - # BM25 filter options - bm25_threshold: float = 1.0 - top_k: int = 10 - - # LLM filter options - llm_config: Optional[Dict[str, Any]] = None - relevance_threshold: float = 0.7 - max_tokens: int = 4000 - - -class MarkdownConfig(BaseModel): - """Configuration for markdown generation.""" - - include_images: bool = True - include_links: bool = True - include_tables: bool = True - include_code: bool = True - max_image_width: int = 800 - link_preview: bool = False - citations: bool = True - content_filter: Optional[ContentFilterConfig] = None - - -class AdaptiveCrawlConfig(BaseModel): - """Configuration for adaptive crawling.""" - - confidence_threshold: float = 0.7 - max_depth: int = 5 - max_pages: int = 20 - strategy: Literal["statistical", "embedding"] = "statistical" - learning_rate: float = 0.1 - min_relevance_score: float = 0.3 - saturation_threshold: int = 5 - quality_threshold: float = 0.5 - save_state: bool = True - state_path: Optional[str] = None - - -class VirtualScrollConfig(BaseModel): - """Configuration for virtual scrolling.""" - - container_selector: Optional[str] = None - scroll_count: int = 10 - scroll_by: Literal["viewport_height", "container_height", "pixels"] = "viewport_height" - scroll_pixels: int = 1000 - wait_after_scroll: float = 2.0 - wait_for_selector: Optional[str] = None - scroll_timeout: float = 30.0 - check_content_changes: bool = True - min_content_increase: int = 100 - max_scroll_attempts: int = 50 - auto_detect_infinite_scroll: bool = True - scroll_pause_detection: bool = True - content_stabilization_time: float = 3.0 - - -class LinkAnalysisConfig(BaseModel): - """Configuration for link analysis.""" - - query: Optional[str] = None - score_threshold: float = 0.3 - concurrent_requests: int = 10 - max_preview_length: int = 500 - enable_domain_authority: bool = True - enable_content_preview: bool = True - enable_freshness_scoring: bool = True - preview_timeout: float = 5.0 - - high_authority_domains: List[str] = Field(default_factory=lambda: [ - 'wikipedia.org', 'github.com', 'stackoverflow.com', - 'mozilla.org', 'w3.org', 'ietf.org', 'arxiv.org' - ]) - - medium_authority_domains: List[str] = Field(default_factory=lambda: [ - 'medium.com', 'dev.to', 'reddit.com', 'news.ycombinator.com' - ]) - - low_quality_indicators: List[str] = Field(default_factory=lambda: [ - 'ads', 'advertisement', 'popup', 'spam', 'click' - ]) diff --git a/apps/backend/app/models/responses.py b/apps/backend/app/models/responses.py deleted file mode 100644 index 7b13348..0000000 --- a/apps/backend/app/models/responses.py +++ /dev/null @@ -1,214 +0,0 @@ -""" -API response models using Pydantic v2. -""" -from typing import Dict, List, Optional, Any, Literal -from datetime import datetime -from pydantic import BaseModel, Field, HttpUrl -from enum import Enum - - -class ServiceHealth(BaseModel): - """Service health status.""" - status: Literal["healthy", "degraded", "unhealthy"] - latency_ms: int - last_check: datetime - details: Optional[Dict[str, Any]] = None - - -class ContentMetadata(BaseModel): - """Extracted content metadata.""" - title: Optional[str] = None - description: Optional[str] = None - author: Optional[str] = None - published_date: Optional[datetime] = None - modified_date: Optional[datetime] = None - keywords: List[str] = Field(default_factory=list) - og_data: Dict[str, str] = Field(default_factory=dict, description="Open Graph data") - twitter_data: Dict[str, str] = Field(default_factory=dict, description="Twitter Card data") - json_ld: Optional[Dict[str, Any]] = Field(default=None, description="JSON-LD structured data") - - -class ScrapedContent(BaseModel): - """Scraped content from a webpage.""" - url: HttpUrl - title: Optional[str] = None - text: str - html: Optional[str] = None - images: List[HttpUrl] = Field(default_factory=list) - links: List[HttpUrl] = Field(default_factory=list) - metadata: ContentMetadata - extraction_success: bool - extraction_time_ms: int - word_count: int - language_detected: Optional[str] = None - content_quality_score: float = Field(ge=0.0, le=1.0, description="Content quality score") - error_message: Optional[str] = None - - -class SearchResult(BaseModel): - """Individual search result.""" - rank: int - title: str - url: HttpUrl - snippet: str - engine: str - score: Optional[float] = Field(default=None, description="Relevance score if available") - scraped_content: Optional[ScrapedContent] = None - cached: bool = False - - class Config: - json_schema_extra = { - "example": { - "rank": 1, - "title": "Python Web Scraping Tutorial", - "url": "https://example.com/python-scraping", - "snippet": "Learn how to scrape websites using Python...", - "engine": "google", - "cached": False - } - } - - -class SearchMetadata(BaseModel): - """Search operation metadata.""" - query: str - engines_used: List[str] - engines_succeeded: List[str] - engines_failed: List[str] = Field(default_factory=list) - total_results_found: int - results_returned: int - search_time_ms: int - timestamp: datetime = Field(default_factory=datetime.utcnow) - - -class UnQuestResponse(BaseModel): - """Main search and scrape response.""" - search_metadata: SearchMetadata - results: List[SearchResult] - processing_time_ms: int - cached: bool - cache_key: Optional[str] = None - total_results: int - request_id: str = Field(description="Unique request identifier") - - class Config: - json_schema_extra = { - "example": { - "search_metadata": { - "query": "Python web scraping", - "engines_used": ["google", "bing"], - "engines_succeeded": ["google", "bing"], - "engines_failed": [], - "total_results_found": 250, - "results_returned": 10, - "search_time_ms": 1250, - "timestamp": "2024-01-01T00:00:00Z" - }, - "results": [], - "processing_time_ms": 2500, - "cached": False, - "total_results": 10, - "request_id": "req_123456" - } - } - - -class AsyncTaskResponse(BaseModel): - """Response for async task creation.""" - task_id: str - status: Literal["pending", "processing", "completed", "failed"] - created_at: datetime - webhook_url: Optional[HttpUrl] = None - estimated_completion_seconds: int = Field(default=60) - - class Config: - json_schema_extra = { - "example": { - "task_id": "task_abc123", - "status": "pending", - "created_at": "2024-01-01T00:00:00Z", - "webhook_url": "https://example.com/webhook", - "estimated_completion_seconds": 30 - } - } - - -class BatchSearchResponse(BaseModel): - """Response for batch search operations.""" - batch_id: str - queries_processed: int - queries_failed: int - results: Dict[str, List[SearchResult]] - processing_time_ms: int - errors: Dict[str, str] = Field(default_factory=dict) - - -class ErrorResponse(BaseModel): - """Standard error response.""" - error: str - message: str - details: Optional[Dict[str, Any]] = None - request_id: Optional[str] = None - timestamp: datetime = Field(default_factory=datetime.utcnow) - - class Config: - json_schema_extra = { - "example": { - "error": "ValidationError", - "message": "Invalid search query", - "details": {"field": "query", "reason": "Query too long"}, - "request_id": "req_123456", - "timestamp": "2024-01-01T00:00:00Z" - } - } - - -class HealthResponse(BaseModel): - """Application health check response.""" - status: Literal["healthy", "degraded", "unhealthy"] - version: str - environment: str - services: Dict[str, ServiceHealth] - timestamp: datetime = Field(default_factory=datetime.utcnow) - uptime_seconds: int - - class Config: - json_schema_extra = { - "example": { - "status": "healthy", - "version": "1.0.0", - "environment": "production", - "services": { - "searxng": { - "status": "healthy", - "latency_ms": 120, - "last_check": "2024-01-01T00:00:00Z" - }, - "redis": { - "status": "healthy", - "latency_ms": 5, - "last_check": "2024-01-01T00:00:00Z" - } - }, - "timestamp": "2024-01-01T00:00:00Z", - "uptime_seconds": 3600 - } - } - - -class EngineInfo(BaseModel): - """Search engine information.""" - name: str - enabled: bool - categories: List[str] - supported_languages: List[str] - safe_search_support: bool - time_range_support: bool - paging_support: bool - - -class EnginesListResponse(BaseModel): - """List of available search engines.""" - engines: Dict[str, EngineInfo] - total_engines: int - enabled_engines: int diff --git a/apps/backend/app/models/users.py b/apps/backend/app/models/users.py deleted file mode 100644 index a074c81..0000000 --- a/apps/backend/app/models/users.py +++ /dev/null @@ -1,351 +0,0 @@ -""" -User, subscription, and billing models for the UnSearch API. -""" -from sqlalchemy import Column, Integer, String, DateTime, Float, Boolean, JSON, Text, Index, ForeignKey, Enum -from sqlalchemy.orm import relationship -from sqlalchemy.sql import func -from sqlalchemy.ext.declarative import declarative_base -from datetime import datetime, timedelta -import enum -import uuid - -Base = declarative_base() - - -class PlanType(enum.Enum): - """Subscription plan types.""" - FREE = "free" - PRO = "pro" - ENTERPRISE = "enterprise" - - -class SubscriptionStatus(enum.Enum): - """Subscription status.""" - ACTIVE = "active" - TRIALING = "trialing" - CANCELLED = "cancelled" - PAST_DUE = "past_due" - UNPAID = "unpaid" - INCOMPLETE = "incomplete" - - -class User(Base): - """User account model.""" - __tablename__ = "users" - - id = Column(Integer, primary_key=True) - uuid = Column(String(36), unique=True, nullable=False, default=lambda: str(uuid.uuid4())) - email = Column(String(255), unique=True, nullable=False, index=True) - username = Column(String(100), unique=True, nullable=True, index=True) - password_hash = Column(String(255), nullable=False) - salt = Column(String(32), nullable=False) - - # Profile - full_name = Column(String(255)) - company = Column(String(255)) - phone = Column(String(20)) - timezone = Column(String(50), default="UTC") - - # Authentication - is_active = Column(Boolean, default=True) - is_verified = Column(Boolean, default=False) - is_admin = Column(Boolean, default=False) - email_verified_at = Column(DateTime(timezone=True)) - verification_token = Column(String(255)) - reset_token = Column(String(255)) - reset_token_expires = Column(DateTime(timezone=True)) - - # Stripe - stripe_customer_id = Column(String(255), unique=True, index=True) - stripe_payment_method_id = Column(String(255)) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), onupdate=func.now()) - last_login_at = Column(DateTime(timezone=True)) - - # Relationships - api_keys = relationship("UserAPIKey", back_populates="user", cascade="all, delete-orphan") - subscriptions = relationship("Subscription", back_populates="user", cascade="all, delete-orphan") - usage_records = relationship("UsageRecord", back_populates="user", cascade="all, delete-orphan") - invoices = relationship("Invoice", back_populates="user", cascade="all, delete-orphan") - - __table_args__ = ( - Index("idx_users_active", "is_active"), - Index("idx_users_stripe", "stripe_customer_id"), - ) - - @property - def current_subscription(self): - """Get the current active subscription.""" - return next( - (sub for sub in self.subscriptions if sub.status == SubscriptionStatus.ACTIVE), - None - ) - - @property - def current_plan(self): - """Get the current plan type.""" - sub = self.current_subscription - return sub.plan_type if sub else PlanType.FREE - - -class UserAPIKey(Base): - """User-specific API keys.""" - __tablename__ = "user_api_keys" - - id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey("users.id"), nullable=False) - key = Column(String(64), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=False) - description = Column(Text) - - # Permissions - scopes = Column(JSON, default=["read", "write"]) # API scopes/permissions - ip_whitelist = Column(JSON) # Optional IP restrictions - - # Usage - last_used_at = Column(DateTime(timezone=True)) - request_count = Column(Integer, default=0) - - # Status - is_active = Column(Boolean, default=True) - expires_at = Column(DateTime(timezone=True)) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - user = relationship("User", back_populates="api_keys") - - __table_args__ = ( - Index("idx_user_api_keys_active", "is_active"), - Index("idx_user_api_keys_user", "user_id"), - ) - - -class Subscription(Base): - """User subscription model.""" - __tablename__ = "subscriptions" - - id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey("users.id"), nullable=False) - - # Stripe - stripe_subscription_id = Column(String(255), unique=True, index=True) - stripe_price_id = Column(String(255)) - stripe_product_id = Column(String(255)) - - # Plan details - plan_type = Column(Enum(PlanType), nullable=False, default=PlanType.FREE) - status = Column(Enum(SubscriptionStatus), nullable=False, default=SubscriptionStatus.ACTIVE) - - # Billing - amount = Column(Float, default=0.0) # Monthly amount in USD - currency = Column(String(3), default="usd") - interval = Column(String(20), default="month") # month, year - - # Limits (cached from plan) - search_limit = Column(Integer, default=1000) # Monthly search limit - scrape_limit = Column(Integer, default=10000) # Monthly scrape limit - rate_limit = Column(String(50), default="100/hour") # Rate limit - - # Features - features = Column(JSON, default={ - "api_access": True, - "webhook_support": False, - "priority_support": False, - "custom_engines": False, - "dedicated_pool": False, - "sla": False - }) - - # Dates - trial_start = Column(DateTime(timezone=True)) - trial_end = Column(DateTime(timezone=True)) - current_period_start = Column(DateTime(timezone=True)) - current_period_end = Column(DateTime(timezone=True)) - cancelled_at = Column(DateTime(timezone=True)) - ended_at = Column(DateTime(timezone=True)) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), onupdate=func.now()) - - # Relationships - user = relationship("User", back_populates="subscriptions") - - __table_args__ = ( - Index("idx_subscriptions_user", "user_id"), - Index("idx_subscriptions_status", "status"), - Index("idx_subscriptions_stripe", "stripe_subscription_id"), - ) - - @property - def is_active(self): - """Check if subscription is currently active.""" - return self.status in [SubscriptionStatus.ACTIVE, SubscriptionStatus.TRIALING] - - @property - def days_remaining(self): - """Days remaining in current period.""" - if self.current_period_end: - delta = self.current_period_end - datetime.utcnow() - return max(0, delta.days) - return 0 - - -class UsageRecord(Base): - """Track API usage per user.""" - __tablename__ = "usage_records" - - id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey("users.id"), nullable=False) - - # Period - period_start = Column(DateTime(timezone=True), nullable=False) - period_end = Column(DateTime(timezone=True), nullable=False) - - # Counts - search_count = Column(Integer, default=0) - scrape_count = Column(Integer, default=0) - api_calls = Column(Integer, default=0) - - # Detailed usage - usage_by_engine = Column(JSON, default={}) # {"google": 100, "bing": 50} - usage_by_day = Column(JSON, default={}) # {"2024-01-01": 50, "2024-01-02": 75} - - # Overages - search_overage = Column(Integer, default=0) - scrape_overage = Column(Integer, default=0) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), onupdate=func.now()) - - # Relationships - user = relationship("User", back_populates="usage_records") - - __table_args__ = ( - Index("idx_usage_records_user", "user_id"), - Index("idx_usage_records_period", "period_start", "period_end"), - Index("idx_usage_user_period", "user_id", "period_start", "period_end", unique=True), - ) - - -class Plan(Base): - """Subscription plans configuration.""" - __tablename__ = "plans" - - id = Column(Integer, primary_key=True) - name = Column(String(100), unique=True, nullable=False) - display_name = Column(String(255), nullable=False) - description = Column(Text) - - # Stripe - stripe_product_id = Column(String(255), unique=True) - stripe_price_id = Column(String(255), unique=True) - - # Pricing - price = Column(Float, nullable=False) # Monthly price in USD - currency = Column(String(3), default="usd") - interval = Column(String(20), default="month") - - # Limits - search_limit = Column(Integer) # null = unlimited - scrape_limit = Column(Integer) # null = unlimited - rate_limit = Column(String(50)) # e.g., "1000/hour" - concurrent_requests = Column(Integer, default=10) - - # Features - features = Column(JSON, default={}) - - # Status - is_active = Column(Boolean, default=True) - is_visible = Column(Boolean, default=True) # Show in pricing page - - # Metadata - extra_metadata = Column(JSON, default={}) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), onupdate=func.now()) - - __table_args__ = ( - Index("idx_plans_active", "is_active"), - Index("idx_plans_stripe", "stripe_product_id", "stripe_price_id"), - ) - - -class Invoice(Base): - """User invoices from Stripe.""" - __tablename__ = "invoices" - - id = Column(Integer, primary_key=True) - user_id = Column(Integer, ForeignKey("users.id"), nullable=False) - - # Stripe - stripe_invoice_id = Column(String(255), unique=True, index=True) - stripe_charge_id = Column(String(255)) - - # Invoice details - invoice_number = Column(String(100), unique=True) - status = Column(String(50)) # draft, open, paid, void, uncollectible - - # Amounts (in cents) - amount_due = Column(Integer) - amount_paid = Column(Integer) - amount_remaining = Column(Integer) - subtotal = Column(Integer) - tax = Column(Integer) - total = Column(Integer) - currency = Column(String(3), default="usd") - - # Dates - period_start = Column(DateTime(timezone=True)) - period_end = Column(DateTime(timezone=True)) - due_date = Column(DateTime(timezone=True)) - paid_at = Column(DateTime(timezone=True)) - - # URLs - invoice_pdf = Column(String(500)) - hosted_invoice_url = Column(String(500)) - - # Metadata - description = Column(Text) - extra_metadata = Column(JSON, default={}) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), onupdate=func.now()) - - # Relationships - user = relationship("User", back_populates="invoices") - - __table_args__ = ( - Index("idx_invoices_user", "user_id"), - Index("idx_invoices_stripe", "stripe_invoice_id"), - Index("idx_invoices_status", "status"), - ) - - -class WebhookEvent(Base): - """Track Stripe webhook events.""" - __tablename__ = "webhook_events" - - id = Column(Integer, primary_key=True) - stripe_event_id = Column(String(255), unique=True, nullable=False, index=True) - event_type = Column(String(100), nullable=False, index=True) - - # Processing - processed = Column(Boolean, default=False) - processed_at = Column(DateTime(timezone=True)) - error_message = Column(Text) - retry_count = Column(Integer, default=0) - - # Data - data = Column(JSON, nullable=False) - - # Timestamps - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - __table_args__ = ( - Index("idx_webhook_events_processed", "processed"), - Index("idx_webhook_events_type", "event_type"), - ) diff --git a/apps/backend/app/services/__init__.py b/apps/backend/app/services/__init__.py deleted file mode 100644 index bb7f454..0000000 --- a/apps/backend/app/services/__init__.py +++ /dev/null @@ -1,384 +0,0 @@ -""" -Enhanced backend services with crawl4ai-inspired capabilities. - -This module provides a comprehensive suite of web scraping and content processing services: - -Core Services: -- ContentScrapingService: Basic web content extraction -- EnhancedScrapingService: Advanced crawling with all features - -Advanced Extraction: -- ExtractionStrategies: LLM, Cosine, JsonCSS, Regex extraction methods -- ContentFilters: BM25, Pruning, LLM content filtering -- ChunkingStrategies: Various text chunking approaches -- TableExtraction: Intelligent table detection and extraction - -Content Processing: -- MarkdownGeneration: Enhanced HTML to markdown conversion -- LinkAnalysis: Intelligent link scoring and analysis - -Crawling Intelligence: -- AdaptiveCrawling: Learning-based crawling optimization -- VirtualScrolling: Infinite scroll page handling -- URLSeeder: Advanced URL discovery from multiple sources - -Infrastructure: -- Dispatcher: Memory-aware concurrent operation management -- BrowserConfig: Comprehensive browser configuration -- RateLimiter: Advanced rate limiting with multiple strategies - -All services are designed to work together seamlessly while maintaining -backward compatibility with existing systems. -""" - -from .scraping import ContentScrapingService -from .enhanced_scraping import EnhancedScrapingService, get_enhanced_scraping_service - -from .extraction_strategies import ( - ExtractionStrategy, NoExtractionStrategy, CosineStrategy, - JsonCssExtractionStrategy, RegexExtractionStrategy, LLMExtractionStrategy, - create_extraction_strategy -) - -from .content_filters import ( - RelevantContentFilter, NoContentFilter, PruningContentFilter, - BM25ContentFilter, LLMContentFilter, create_content_filter -) - -from .chunking_strategies import ( - ChunkingStrategy, IdentityChunking, RegexChunking, SentenceChunking, - ParagraphChunking, FixedSizeChunking, TopicChunking, HybridChunking, - create_chunking_strategy, chunk_text, smart_chunk_for_llm -) - -from .table_extraction import ( - TableExtractionStrategy, NoTableExtraction, DefaultTableExtraction, - LLMTableExtraction, SmartTableExtraction, create_table_extraction_strategy, - extract_tables, tables_to_markdown -) - -from .markdown_generation import ( - MarkdownGenerationStrategy, DefaultMarkdownGenerator, - generate_markdown, generate_simple_markdown -) - -from .link_analysis import ( - LinkInfo, LinkScorer, LinkAnalyzer, analyze_links, get_top_links -) - -from .adaptive_crawling import ( - CrawlState, CrawlStrategy, StatisticalStrategy, AdaptiveCrawler, - create_adaptive_crawler -) - -from .virtual_scrolling import ( - VirtualScrollConfig, VirtualScrollHandler, PuppeteerVirtualScroller, - create_virtual_scroll_config, scroll_infinite_page -) - -from .url_seeder import ( - SeedingConfig, DiscoveredURL, URLSeeder, discover_urls, filter_urls_by_patterns -) - -from .browser_config import ( - BrowserType, DeviceType, GeolocationConfig, ProxyConfig, UserAgentConfig, - BrowserConfig, get_stealth_browser_config, get_mobile_browser_config, - get_high_performance_browser_config, create_browser_config_from_env -) - -from .dispatcher import ( - BaseDispatcher, SemaphoreDispatcher, MemoryAdaptiveDispatcher, - RateLimiter, TaskResult, DispatchStats, create_dispatcher, create_rate_limiter -) - -# NEW: Missing crawl4ai features implemented -from .deep_crawling import ( - DeepCrawlStrategy, BFSDeepCrawlStrategy, DFSDeepCrawlStrategy, BestFirstCrawlStrategy, - URLFilter, DomainFilter, URLPatternFilter, ContentTypeFilter, SEOFilter, ContentRelevanceFilter, - FilterChain, URLScorer, KeywordRelevanceScorer, PathDepthScorer, DomainAuthorityScorer, - FreshnessScorer, CompositeScorer, create_deep_crawl_strategy, deep_crawl -) - -from .pdf_processing import ( - PDFProcessorStrategy, MockPDFProcessor, NaivePDFProcessor, PDFMetadata, PDFPage, - PDFProcessResult, PDFImage, create_pdf_processor, process_pdf_file, process_pdf_bytes, - extract_pdf_text, pdf_to_markdown -) - -from .browser_profiler import ( - BrowserProfiler, BrowserProfile, get_browser_profiler, create_browser_profile, - get_profile_browser_config, list_browser_profiles -) - -from .link_preview import ( - LinkPreview, LinkPreviewConfig, LinkPreviewResult, LinkMetadata, - extract_link_previews, filter_links_by_quality -) - -from .crawler_monitor import ( - CrawlerMonitor, CrawlStatus, TaskMetrics, SystemMetrics, CrawlerStats, - create_crawler_monitor, get_global_monitor, start_global_monitoring, stop_global_monitoring -) - -from .proxy_rotation import ( - ProxyRotationStrategy, RoundRobinProxyStrategy, RandomProxyStrategy, - WeightedProxyStrategy, GeographicProxyStrategy, ProxyStatus, ProxyInfo, ProxyMetrics, - create_proxy_strategy, create_proxy_list_from_strings, test_proxy_rotation -) - -# NEW: Additional missing crawl4ai components -from .database_manager import ( - DatabaseManager, CrawlRecord, DatabaseStats, get_database_manager, - store_crawl_data, get_cached_content, search_content -) - -from .cache_context import ( - CacheContext, CacheContextManager, CacheMode, URLType, CacheRule, CacheStats, - get_cache_manager, create_cache_context, should_cache_url -) - -from .user_agent_generator import ( - UAGenerator, ValidUAGenerator, OnlineUAGenerator, CustomUAGenerator, - UserAgentManager, UserAgentProfile, get_user_agent_manager, - generate_user_agent, get_random_user_agent, get_user_agent_with_hints -) - -from .html_converter import ( - HTMLToTextConverter, HTMLToMarkdownConverter, ConversionConfig, - create_html_converter, html_to_text, html_to_markdown, extract_clean_text -) - -# Legacy imports for backward compatibility -from .auth_service import AuthService -from .cache import CacheService, get_cache_service -from .database import DatabaseService -from .searxng import SearxngService -from .puppeteer_client import PuppeteerClient - - -__all__ = [ - # Core services - "ContentScrapingService", - "EnhancedScrapingService", - "get_enhanced_scraping_service", - - # Extraction strategies - "ExtractionStrategy", - "NoExtractionStrategy", - "CosineStrategy", - "JsonCssExtractionStrategy", - "RegexExtractionStrategy", - "LLMExtractionStrategy", - "create_extraction_strategy", - - # Content filters - "RelevantContentFilter", - "NoContentFilter", - "PruningContentFilter", - "BM25ContentFilter", - "LLMContentFilter", - "create_content_filter", - - # Chunking strategies - "ChunkingStrategy", - "IdentityChunking", - "RegexChunking", - "SentenceChunking", - "ParagraphChunking", - "FixedSizeChunking", - "TopicChunking", - "HybridChunking", - "create_chunking_strategy", - "chunk_text", - "smart_chunk_for_llm", - - # Table extraction - "TableExtractionStrategy", - "NoTableExtraction", - "DefaultTableExtraction", - "LLMTableExtraction", - "SmartTableExtraction", - "create_table_extraction_strategy", - "extract_tables", - "tables_to_markdown", - - # Markdown generation - "MarkdownGenerationStrategy", - "DefaultMarkdownGenerator", - "generate_markdown", - "generate_simple_markdown", - - # Link analysis - "LinkInfo", - "LinkScorer", - "LinkAnalyzer", - "analyze_links", - "get_top_links", - - # Adaptive crawling - "CrawlState", - "CrawlStrategy", - "StatisticalStrategy", - "AdaptiveCrawler", - "create_adaptive_crawler", - - # Virtual scrolling - "VirtualScrollConfig", - "VirtualScrollHandler", - "PuppeteerVirtualScroller", - "create_virtual_scroll_config", - "scroll_infinite_page", - - # URL seeding - "SeedingConfig", - "DiscoveredURL", - "URLSeeder", - "discover_urls", - "filter_urls_by_patterns", - - # Browser configuration - "BrowserType", - "DeviceType", - "GeolocationConfig", - "ProxyConfig", - "UserAgentConfig", - "BrowserConfig", - "get_stealth_browser_config", - "get_mobile_browser_config", - "get_high_performance_browser_config", - "create_browser_config_from_env", - - # Dispatcher system - "BaseDispatcher", - "SemaphoreDispatcher", - "MemoryAdaptiveDispatcher", - "RateLimiter", - "TaskResult", - "DispatchStats", - "create_dispatcher", - "create_rate_limiter", - - # Deep crawling system - "DeepCrawlStrategy", - "BFSDeepCrawlStrategy", - "DFSDeepCrawlStrategy", - "BestFirstCrawlStrategy", - "URLFilter", - "DomainFilter", - "URLPatternFilter", - "ContentTypeFilter", - "SEOFilter", - "ContentRelevanceFilter", - "FilterChain", - "URLScorer", - "KeywordRelevanceScorer", - "PathDepthScorer", - "DomainAuthorityScorer", - "FreshnessScorer", - "CompositeScorer", - "create_deep_crawl_strategy", - "deep_crawl", - - # PDF processing - "PDFProcessorStrategy", - "MockPDFProcessor", - "NaivePDFProcessor", - "PDFMetadata", - "PDFPage", - "PDFProcessResult", - "PDFImage", - "create_pdf_processor", - "process_pdf_file", - "process_pdf_bytes", - "extract_pdf_text", - "pdf_to_markdown", - - # Browser profiling - "BrowserProfiler", - "BrowserProfile", - "get_browser_profiler", - "create_browser_profile", - "get_profile_browser_config", - "list_browser_profiles", - - # Link preview system - "LinkPreview", - "LinkPreviewConfig", - "LinkPreviewResult", - "LinkMetadata", - "extract_link_previews", - "filter_links_by_quality", - - # Crawler monitoring - "CrawlerMonitor", - "CrawlStatus", - "TaskMetrics", - "SystemMetrics", - "CrawlerStats", - "create_crawler_monitor", - "get_global_monitor", - "start_global_monitoring", - "stop_global_monitoring", - - # Proxy rotation strategies - "ProxyRotationStrategy", - "RoundRobinProxyStrategy", - "RandomProxyStrategy", - "WeightedProxyStrategy", - "GeographicProxyStrategy", - "ProxyStatus", - "ProxyInfo", - "ProxyMetrics", - "create_proxy_strategy", - "create_proxy_list_from_strings", - "test_proxy_rotation", - - # Database management - "DatabaseManager", - "CrawlRecord", - "DatabaseStats", - "get_database_manager", - "store_crawl_data", - "get_cached_content", - "search_content", - - # Cache context management - "CacheContext", - "CacheContextManager", - "CacheMode", - "URLType", - "CacheRule", - "CacheStats", - "get_cache_manager", - "create_cache_context", - "should_cache_url", - - # User agent generation - "UAGenerator", - "ValidUAGenerator", - "OnlineUAGenerator", - "CustomUAGenerator", - "UserAgentManager", - "UserAgentProfile", - "get_user_agent_manager", - "generate_user_agent", - "get_random_user_agent", - "get_user_agent_with_hints", - - # HTML conversion - "HTMLToTextConverter", - "HTMLToMarkdownConverter", - "ConversionConfig", - "create_html_converter", - "html_to_text", - "html_to_markdown", - "extract_clean_text", - - # Legacy services - "AuthService", - "CacheService", - "get_cache_service", - "DatabaseService", - "SearxngService", - "PuppeteerClient" -] \ No newline at end of file diff --git a/apps/backend/app/services/actions_system.py b/apps/backend/app/services/actions_system.py deleted file mode 100644 index 386510c..0000000 --- a/apps/backend/app/services/actions_system.py +++ /dev/null @@ -1,575 +0,0 @@ -""" -Advanced Actions System for browser automation inspired by Firecrawl. - -Provides sophisticated browser automation capabilities including: -- Click, scroll, input, wait actions -- Screenshot capture -- JavaScript execution -- PDF generation -- Complex interaction sequences -""" - -import asyncio -import time -import json -import base64 -from typing import Dict, List, Optional, Any, Union, Literal -from dataclasses import dataclass, field -from enum import Enum -import structlog - -from app.config import get_settings -from app.models.responses import ScrapedContent - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class ActionType(Enum): - """Types of browser actions.""" - WAIT = "wait" - CLICK = "click" - SCROLL = "scroll" - WRITE = "write" - PRESS = "press" - SCREENSHOT = "screenshot" - SCRAPE = "scrape" - EXECUTE_JAVASCRIPT = "executeJavascript" - PDF = "pdf" - - -@dataclass -class WaitAction: - """Wait action configuration.""" - type: Literal["wait"] = "wait" - milliseconds: Optional[int] = None - selector: Optional[str] = None - - def __post_init__(self): - if not self.milliseconds and not self.selector: - raise ValueError("Either milliseconds or selector must be provided") - if self.milliseconds and self.selector: - raise ValueError("Only one of milliseconds or selector can be provided") - - -@dataclass -class ClickAction: - """Click action configuration.""" - type: Literal["click"] = "click" - selector: str - all: bool = False - - -@dataclass -class ScrollAction: - """Scroll action configuration.""" - type: Literal["scroll"] = "scroll" - direction: Literal["up", "down"] = "down" - selector: Optional[str] = None - - -@dataclass -class WriteAction: - """Write text action configuration.""" - type: Literal["write"] = "write" - text: str - - -@dataclass -class PressAction: - """Press key action configuration.""" - type: Literal["press"] = "press" - key: str - - -@dataclass -class ScreenshotAction: - """Screenshot action configuration.""" - type: Literal["screenshot"] = "screenshot" - fullPage: bool = False - quality: Optional[int] = None # 1-100 - viewport: Optional[Dict[str, int]] = None - - -@dataclass -class ScrapeAction: - """Scrape current state action configuration.""" - type: Literal["scrape"] = "scrape" - - -@dataclass -class ExecuteJavaScriptAction: - """Execute JavaScript action configuration.""" - type: Literal["executeJavascript"] = "executeJavascript" - script: str - - -@dataclass -class PDFAction: - """Generate PDF action configuration.""" - type: Literal["pdf"] = "pdf" - landscape: bool = False - scale: float = 1.0 - format: Literal["A0", "A1", "A2", "A3", "A4", "A5", "A6", "Letter", "Legal", "Tabloid", "Ledger"] = "Letter" - - -# Union type for all actions -Action = Union[ - WaitAction, - ClickAction, - ScrollAction, - WriteAction, - PressAction, - ScreenshotAction, - ScrapeAction, - ExecuteJavaScriptAction, - PDFAction -] - - -@dataclass -class ActionResult: - """Result of an action execution.""" - action_type: str - success: bool - data: Optional[Any] = None - error: Optional[str] = None - execution_time_ms: int = 0 - screenshot_url: Optional[str] = None - - -@dataclass -class ActionsSequenceResult: - """Result of executing a sequence of actions.""" - success: bool - actions_results: List[ActionResult] = field(default_factory=list) - screenshots: List[str] = field(default_factory=list) - scrapes: List[Dict[str, Any]] = field(default_factory=list) - javascript_returns: List[Dict[str, Any]] = field(default_factory=list) - pdfs: List[str] = field(default_factory=list) - total_execution_time_ms: int = 0 - error: Optional[str] = None - - -class BrowserActionsExecutor: - """ - Browser actions executor using Playwright or Puppeteer. - - Executes complex sequences of browser actions including: - - User interactions (click, scroll, type) - - Waiting for elements or time - - Screenshot capture - - JavaScript execution - - PDF generation - - Content scraping at various stages - """ - - def __init__(self): - """Initialize the browser actions executor.""" - self.browser = None - self.page = None - self.context = None - self.actions_stats = {"total_actions": 0, "successful_actions": 0} - - async def execute_actions_sequence( - self, - url: str, - actions: List[Dict[str, Any]], - browser_options: Optional[Dict[str, Any]] = None - ) -> ActionsSequenceResult: - """ - Execute a sequence of browser actions. - - Args: - url: URL to navigate to - actions: List of action dictionaries - browser_options: Browser configuration options - - Returns: - ActionsSequenceResult with execution results - """ - start_time = time.time() - result = ActionsSequenceResult(success=True) - - try: - # Parse actions - parsed_actions = self._parse_actions(actions) - - # Initialize browser - await self._initialize_browser(browser_options or {}) - - # Navigate to URL - await self._navigate_to_url(url) - - # Execute each action - for i, action in enumerate(parsed_actions): - try: - logger.info("executing_action", - index=i, - action_type=action.type, - url=url) - - action_result = await self._execute_single_action(action) - result.actions_results.append(action_result) - - # Collect results by type - if action_result.success: - self.actions_stats["successful_actions"] += 1 - - if action.type == "screenshot" and action_result.screenshot_url: - result.screenshots.append(action_result.screenshot_url) - elif action.type == "scrape" and action_result.data: - result.scrapes.append(action_result.data) - elif action.type == "executeJavascript" and action_result.data: - result.javascript_returns.append({ - "type": type(action_result.data).__name__, - "value": action_result.data - }) - elif action.type == "pdf" and action_result.data: - result.pdfs.append(action_result.data) - - self.actions_stats["total_actions"] += 1 - - except Exception as e: - error_msg = f"Action {i} ({action.type}) failed: {str(e)}" - logger.error("action_execution_failed", - index=i, - action_type=action.type, - error=str(e)) - - result.actions_results.append(ActionResult( - action_type=action.type, - success=False, - error=error_msg - )) - - # Continue with other actions unless critical failure - if action.type in ["wait", "click", "write"]: - continue - else: - result.success = False - result.error = error_msg - break - - result.total_execution_time_ms = int((time.time() - start_time) * 1000) - - logger.info("actions_sequence_completed", - url=url, - total_actions=len(actions), - successful_actions=len([r for r in result.actions_results if r.success]), - execution_time_ms=result.total_execution_time_ms) - - return result - - except Exception as e: - result.success = False - result.error = str(e) - result.total_execution_time_ms = int((time.time() - start_time) * 1000) - - logger.error("actions_sequence_failed", - url=url, - error=str(e), - execution_time_ms=result.total_execution_time_ms) - - return result - - finally: - await self._cleanup_browser() - - def _parse_actions(self, actions: List[Dict[str, Any]]) -> List[Action]: - """Parse action dictionaries into typed action objects.""" - parsed_actions = [] - - for action_dict in actions: - action_type = action_dict.get("type") - - try: - if action_type == "wait": - parsed_actions.append(WaitAction(**action_dict)) - elif action_type == "click": - parsed_actions.append(ClickAction(**action_dict)) - elif action_type == "scroll": - parsed_actions.append(ScrollAction(**action_dict)) - elif action_type == "write": - parsed_actions.append(WriteAction(**action_dict)) - elif action_type == "press": - parsed_actions.append(PressAction(**action_dict)) - elif action_type == "screenshot": - parsed_actions.append(ScreenshotAction(**action_dict)) - elif action_type == "scrape": - parsed_actions.append(ScrapeAction(**action_dict)) - elif action_type == "executeJavascript": - parsed_actions.append(ExecuteJavaScriptAction(**action_dict)) - elif action_type == "pdf": - parsed_actions.append(PDFAction(**action_dict)) - else: - raise ValueError(f"Unknown action type: {action_type}") - - except Exception as e: - logger.error("action_parsing_failed", action=action_dict, error=str(e)) - raise ValueError(f"Invalid action configuration: {str(e)}") - - return parsed_actions - - async def _initialize_browser(self, browser_options: Dict[str, Any]): - """Initialize browser instance.""" - try: - # Try Playwright first - playwright_url = getattr(settings, 'playwright_service_url', None) - if playwright_url: - await self._initialize_playwright_browser(browser_options) - return - - # Fallback to Fire Engine - fire_engine_url = getattr(settings, 'fire_engine_url', None) - if fire_engine_url: - await self._initialize_fire_engine_browser(browser_options) - return - - # Fallback to local Playwright - await self._initialize_local_playwright(browser_options) - - except Exception as e: - logger.error("browser_initialization_failed", error=str(e)) - raise e - - async def _initialize_playwright_browser(self, options: Dict[str, Any]): - """Initialize Playwright browser via service.""" - # This would connect to Playwright service - # For now, simulate browser initialization - self.browser = "playwright_service" - self.page = "playwright_page" - self.context = "playwright_context" - - async def _initialize_fire_engine_browser(self, options: Dict[str, Any]): - """Initialize Fire Engine browser.""" - # This would connect to Fire Engine - # For now, simulate browser initialization - self.browser = "fire_engine" - self.page = "fire_engine_page" - self.context = "fire_engine_context" - - async def _initialize_local_playwright(self, options: Dict[str, Any]): - """Initialize local Playwright browser.""" - try: - from playwright.async_api import async_playwright - - self.playwright = await async_playwright().start() - self.browser = await self.playwright.chromium.launch( - headless=options.get("headless", True), - args=options.get("args", []) - ) - self.context = await self.browser.new_context( - viewport=options.get("viewport", {"width": 1920, "height": 1080}), - user_agent=options.get("userAgent", settings.scraping_user_agent) - ) - self.page = await self.context.new_page() - - except ImportError: - logger.warning("playwright_not_available", fallback="mock") - # Mock browser for testing - self.browser = "mock_browser" - self.page = "mock_page" - self.context = "mock_context" - - async def _navigate_to_url(self, url: str): - """Navigate to the specified URL.""" - if self.page == "mock_page": - return # Mock navigation - - # Here we would navigate using actual browser - # For now, log the navigation - logger.info("navigating_to_url", url=url) - - async def _execute_single_action(self, action: Action) -> ActionResult: - """Execute a single browser action.""" - start_time = time.time() - - try: - if action.type == "wait": - result_data = await self._execute_wait(action) - elif action.type == "click": - result_data = await self._execute_click(action) - elif action.type == "scroll": - result_data = await self._execute_scroll(action) - elif action.type == "write": - result_data = await self._execute_write(action) - elif action.type == "press": - result_data = await self._execute_press(action) - elif action.type == "screenshot": - result_data = await self._execute_screenshot(action) - elif action.type == "scrape": - result_data = await self._execute_scrape(action) - elif action.type == "executeJavascript": - result_data = await self._execute_javascript(action) - elif action.type == "pdf": - result_data = await self._execute_pdf(action) - else: - raise ValueError(f"Unsupported action type: {action.type}") - - execution_time = int((time.time() - start_time) * 1000) - - return ActionResult( - action_type=action.type, - success=True, - data=result_data, - execution_time_ms=execution_time - ) - - except Exception as e: - execution_time = int((time.time() - start_time) * 1000) - - return ActionResult( - action_type=action.type, - success=False, - error=str(e), - execution_time_ms=execution_time - ) - - async def _execute_wait(self, action: WaitAction) -> Any: - """Execute wait action.""" - if action.milliseconds: - await asyncio.sleep(action.milliseconds / 1000) - return {"waited_ms": action.milliseconds} - elif action.selector: - # Wait for element to appear - # In real implementation, would wait for selector - await asyncio.sleep(1) # Simulate wait - return {"waited_for_selector": action.selector} - - async def _execute_click(self, action: ClickAction) -> Any: - """Execute click action.""" - # In real implementation, would click element - await asyncio.sleep(0.1) # Simulate click - return {"clicked_selector": action.selector, "all": action.all} - - async def _execute_scroll(self, action: ScrollAction) -> Any: - """Execute scroll action.""" - # In real implementation, would scroll page - await asyncio.sleep(0.1) # Simulate scroll - return {"scrolled": action.direction, "selector": action.selector} - - async def _execute_write(self, action: WriteAction) -> Any: - """Execute write text action.""" - # In real implementation, would type text - await asyncio.sleep(len(action.text) * 0.01) # Simulate typing - return {"written_text": action.text} - - async def _execute_press(self, action: PressAction) -> Any: - """Execute key press action.""" - # In real implementation, would press key - await asyncio.sleep(0.1) # Simulate key press - return {"pressed_key": action.key} - - async def _execute_screenshot(self, action: ScreenshotAction) -> Any: - """Execute screenshot action.""" - # In real implementation, would capture screenshot - await asyncio.sleep(0.5) # Simulate screenshot capture - - # Generate mock base64 screenshot data - screenshot_data = base64.b64encode(b"mock_screenshot_data").decode() - - return { - "screenshot_base64": screenshot_data, - "fullPage": action.fullPage, - "quality": action.quality - } - - async def _execute_scrape(self, action: ScrapeAction) -> Any: - """Execute scrape current state action.""" - # In real implementation, would scrape current page - await asyncio.sleep(0.2) # Simulate scraping - - return { - "html": "Mock scraped content", - "title": "Mock Page Title", - "text": "Mock page text content", - "timestamp": time.time() - } - - async def _execute_javascript(self, action: ExecuteJavaScriptAction) -> Any: - """Execute JavaScript action.""" - # In real implementation, would execute JavaScript - await asyncio.sleep(0.1) # Simulate JS execution - - # Mock JavaScript result - return { - "script": action.script, - "result": "Mock JS execution result" - } - - async def _execute_pdf(self, action: PDFAction) -> Any: - """Execute PDF generation action.""" - # In real implementation, would generate PDF - await asyncio.sleep(1.0) # Simulate PDF generation - - # Generate mock PDF data - pdf_data = base64.b64encode(b"mock_pdf_data").decode() - - return { - "pdf_base64": pdf_data, - "landscape": action.landscape, - "scale": action.scale, - "format": action.format - } - - async def _cleanup_browser(self): - """Cleanup browser resources.""" - try: - if hasattr(self, 'playwright') and self.playwright: - if self.browser and hasattr(self.browser, 'close'): - await self.browser.close() - await self.playwright.stop() - - # Reset browser state - self.browser = None - self.page = None - self.context = None - - except Exception as e: - logger.warning("browser_cleanup_failed", error=str(e)) - - async def get_actions_stats(self) -> Dict[str, Any]: - """Get actions execution statistics.""" - return { - "actions_stats": self.actions_stats, - "success_rate": ( - self.actions_stats["successful_actions"] / - max(1, self.actions_stats["total_actions"]) - ) if self.actions_stats["total_actions"] > 0 else 0 - } - - -# Convenience functions -async def execute_browser_actions( - url: str, - actions: List[Dict[str, Any]], - browser_options: Optional[Dict[str, Any]] = None -) -> ActionsSequenceResult: - """ - Execute browser actions sequence. - - Args: - url: URL to navigate to - actions: List of action configurations - browser_options: Browser configuration options - - Returns: - ActionsSequenceResult with execution results - """ - executor = BrowserActionsExecutor() - return await executor.execute_actions_sequence(url, actions, browser_options) - - -# Singleton service -_actions_service: Optional[BrowserActionsExecutor] = None - - -async def get_actions_service() -> BrowserActionsExecutor: - """Get or create browser actions service instance.""" - global _actions_service - - if _actions_service is None: - _actions_service = BrowserActionsExecutor() - - return _actions_service diff --git a/apps/backend/app/services/adaptive_crawling.py b/apps/backend/app/services/adaptive_crawling.py deleted file mode 100644 index a1f7f2a..0000000 --- a/apps/backend/app/services/adaptive_crawling.py +++ /dev/null @@ -1,611 +0,0 @@ -""" -Adaptive crawling with learning algorithms inspired by crawl4ai. - -This module implements adaptive information foraging for efficient web crawling: -- Statistical strategy for learning content patterns -- Embedding-based strategy for semantic understanding -- Learning algorithms that improve extraction over time -- State persistence for continued learning -""" - -import asyncio -import json -import math -import pickle -import re -from abc import ABC, abstractmethod -from collections import Counter, defaultdict -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union -from urllib.parse import urljoin, urlparse - -import numpy as np -import structlog -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.metrics.pairwise import cosine_similarity - -from app.models.responses import ScrapedContent -from app.utils.text_processing import clean_tokens, sanitize_text - -logger = structlog.get_logger(__name__) - - -@dataclass -class CrawlState: - """Tracks the current state of adaptive crawling.""" - crawled_urls: Set[str] = field(default_factory=set) - knowledge_base: List[Dict[str, Any]] = field(default_factory=list) - pending_links: List[Dict[str, Any]] = field(default_factory=list) - query: str = "" - metrics: Dict[str, float] = field(default_factory=dict) - - # Statistical tracking - term_frequencies: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - document_frequencies: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - documents_with_terms: Dict[str, Set[int]] = field(default_factory=lambda: defaultdict(set)) - total_documents: int = 0 - - # History tracking for saturation detection - new_terms_history: List[int] = field(default_factory=list) - crawl_order: List[str] = field(default_factory=list) - - # Content quality tracking - quality_scores: List[float] = field(default_factory=list) - relevance_scores: List[float] = field(default_factory=list) - - # Learning parameters - learning_rate: float = 0.1 - confidence_threshold: float = 0.7 - max_crawl_depth: int = 5 - max_pages: int = 20 - - def save(self, path: Union[str, Path]): - """Save state to disk for persistence.""" - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - - # Convert sets to lists for JSON serialization - state_dict = { - 'crawled_urls': list(self.crawled_urls), - 'knowledge_base': self.knowledge_base, - 'pending_links': self.pending_links, - 'query': self.query, - 'metrics': self.metrics, - 'term_frequencies': dict(self.term_frequencies), - 'document_frequencies': dict(self.document_frequencies), - 'documents_with_terms': {k: list(v) for k, v in self.documents_with_terms.items()}, - 'total_documents': self.total_documents, - 'new_terms_history': self.new_terms_history, - 'crawl_order': self.crawl_order, - 'quality_scores': self.quality_scores, - 'relevance_scores': self.relevance_scores, - 'learning_rate': self.learning_rate, - 'confidence_threshold': self.confidence_threshold, - 'max_crawl_depth': self.max_crawl_depth, - 'max_pages': self.max_pages - } - - with open(path, 'w') as f: - json.dump(state_dict, f, indent=2) - - @classmethod - def load(cls, path: Union[str, Path]) -> 'CrawlState': - """Load state from disk.""" - path = Path(path) - if not path.exists(): - return cls() - - with open(path, 'r') as f: - state_dict = json.load(f) - - state = cls() - state.crawled_urls = set(state_dict.get('crawled_urls', [])) - state.knowledge_base = state_dict.get('knowledge_base', []) - state.pending_links = state_dict.get('pending_links', []) - state.query = state_dict.get('query', '') - state.metrics = state_dict.get('metrics', {}) - state.term_frequencies = defaultdict(int, state_dict.get('term_frequencies', {})) - state.document_frequencies = defaultdict(int, state_dict.get('document_frequencies', {})) - state.documents_with_terms = defaultdict( - set, - {k: set(v) for k, v in state_dict.get('documents_with_terms', {}).items()} - ) - state.total_documents = state_dict.get('total_documents', 0) - state.new_terms_history = state_dict.get('new_terms_history', []) - state.crawl_order = state_dict.get('crawl_order', []) - state.quality_scores = state_dict.get('quality_scores', []) - state.relevance_scores = state_dict.get('relevance_scores', []) - state.learning_rate = state_dict.get('learning_rate', 0.1) - state.confidence_threshold = state_dict.get('confidence_threshold', 0.7) - state.max_crawl_depth = state_dict.get('max_crawl_depth', 5) - state.max_pages = state_dict.get('max_pages', 20) - - return state - - -@dataclass -class AdaptiveConfig: - """Configuration for adaptive crawling.""" - confidence_threshold: float = 0.7 - max_depth: int = 5 - max_pages: int = 20 - strategy: str = "statistical" # "statistical" or "embedding" - learning_rate: float = 0.1 - min_relevance_score: float = 0.3 - saturation_threshold: int = 5 # Number of crawls without new terms - quality_threshold: float = 0.5 - save_state: bool = True - state_path: Optional[str] = None - - -class CrawlStrategy(ABC): - """Abstract base class for crawling strategies.""" - - @abstractmethod - async def should_continue_crawling(self, state: CrawlState) -> bool: - """Determine if crawling should continue based on current state.""" - pass - - @abstractmethod - async def score_url(self, url: str, state: CrawlState) -> float: - """Score a URL for crawling priority.""" - pass - - @abstractmethod - async def update_state(self, state: CrawlState, content: ScrapedContent) -> None: - """Update crawling state with new content.""" - pass - - -class StatisticalStrategy(CrawlStrategy): - """ - Statistical strategy for adaptive crawling. - - Uses term frequency analysis, document frequency tracking, - and information saturation detection to determine when - sufficient information has been gathered. - """ - - def __init__(self, config: AdaptiveConfig): - """Initialize statistical strategy.""" - self.config = config - - async def should_continue_crawling(self, state: CrawlState) -> bool: - """Determine if crawling should continue based on statistical analysis.""" - # Check basic limits - if len(state.crawled_urls) >= self.config.max_pages: - logger.info("crawl_limit_reached", pages=len(state.crawled_urls)) - return False - - if len(state.crawl_order) >= self.config.max_depth: - logger.info("depth_limit_reached", depth=len(state.crawl_order)) - return False - - # Check information saturation - if len(state.new_terms_history) >= self.config.saturation_threshold: - recent_new_terms = sum(state.new_terms_history[-self.config.saturation_threshold:]) - avg_new_terms = recent_new_terms / self.config.saturation_threshold - - if avg_new_terms < 5: # Less than 5 new terms per crawl on average - logger.info("information_saturation_detected", avg_new_terms=avg_new_terms) - return False - - # Check confidence threshold - if state.metrics.get('confidence', 0) >= self.config.confidence_threshold: - logger.info("confidence_threshold_reached", confidence=state.metrics['confidence']) - return False - - # Check content quality trends - if len(state.quality_scores) >= 3: - recent_quality = np.mean(state.quality_scores[-3:]) - if recent_quality < self.config.quality_threshold: - logger.info("quality_threshold_not_met", recent_quality=recent_quality) - return False - - return True - - async def score_url(self, url: str, state: CrawlState) -> float: - """Score URL based on statistical relevance.""" - score = 0.5 # Base score - - # Parse URL for features - parsed_url = urlparse(url) - domain = parsed_url.netloc - path = parsed_url.path.lower() - - # Query-based scoring - if state.query: - query_terms = set(clean_tokens(state.query.lower().split())) - - # Check URL path for query terms - url_terms = set(clean_tokens(re.findall(r'[a-zA-Z]+', path))) - term_overlap = len(query_terms.intersection(url_terms)) - if term_overlap > 0: - score += 0.3 * (term_overlap / len(query_terms)) - - # Domain authority (simplified) - authority_domains = [ - 'wikipedia.org', 'github.com', 'stackoverflow.com', - 'news.ycombinator.com', 'reddit.com', 'medium.com' - ] - if any(auth_domain in domain for auth_domain in authority_domains): - score += 0.2 - - # Path depth penalty (prefer shallower pages) - path_depth = len([p for p in path.split('/') if p]) - if path_depth > 0: - score -= min(0.3, path_depth * 0.05) - - # Content type indicators - content_indicators = ['article', 'post', 'blog', 'news', 'tutorial', 'guide'] - if any(indicator in path for indicator in content_indicators): - score += 0.2 - - # Avoid non-content URLs - avoid_patterns = ['login', 'register', 'cart', 'checkout', 'admin', 'api'] - if any(pattern in path for pattern in avoid_patterns): - score -= 0.4 - - return max(0.0, min(1.0, score)) - - async def update_state(self, state: CrawlState, content: ScrapedContent) -> None: - """Update statistical state with new content.""" - if not content.extraction_success or not content.text: - return - - # Tokenize content - tokens = clean_tokens(content.text.lower().split()) - if not tokens: - return - - # Update document tracking - doc_id = state.total_documents - state.total_documents += 1 - - # Track new terms - new_terms_count = 0 - term_counts = Counter(tokens) - - for term, count in term_counts.items(): - # Update term frequencies - state.term_frequencies[term] += count - - # Update document frequencies - if doc_id not in state.documents_with_terms[term]: - state.documents_with_terms[term].add(doc_id) - state.document_frequencies[term] += 1 - - # Count as new term if first occurrence - if state.document_frequencies[term] == 1: - new_terms_count += 1 - - # Update new terms history - state.new_terms_history.append(new_terms_count) - if len(state.new_terms_history) > 20: # Keep last 20 crawls - state.new_terms_history = state.new_terms_history[-20:] - - # Update crawl order - state.crawl_order.append(content.url) - - # Store content in knowledge base - state.knowledge_base.append({ - 'url': content.url, - 'title': content.title, - 'text': content.text[:1000], # Store first 1000 chars - 'quality_score': content.content_quality_score or 0.0, - 'word_count': content.word_count, - 'extraction_time': content.extraction_time_ms, - 'crawl_order': len(state.crawl_order) - }) - - # Update quality tracking - state.quality_scores.append(content.content_quality_score or 0.0) - if len(state.quality_scores) > 20: - state.quality_scores = state.quality_scores[-20:] - - # Calculate relevance score if query exists - if state.query: - relevance = await self._calculate_relevance(content.text, state.query) - state.relevance_scores.append(relevance) - if len(state.relevance_scores) > 20: - state.relevance_scores = state.relevance_scores[-20:] - - # Update confidence metrics - await self._update_confidence_metrics(state) - - async def _calculate_relevance(self, text: str, query: str) -> float: - """Calculate relevance score between text and query.""" - if not text or not query: - return 0.0 - - text_tokens = set(clean_tokens(text.lower().split())) - query_tokens = set(clean_tokens(query.lower().split())) - - if not text_tokens or not query_tokens: - return 0.0 - - # Calculate Jaccard similarity - intersection = len(text_tokens.intersection(query_tokens)) - union = len(text_tokens.union(query_tokens)) - - return intersection / union if union > 0 else 0.0 - - async def _update_confidence_metrics(self, state: CrawlState) -> None: - """Update confidence metrics based on current state.""" - if state.total_documents == 0: - state.metrics['confidence'] = 0.0 - return - - # Calculate information gain rate - if len(state.new_terms_history) > 1: - recent_gains = state.new_terms_history[-3:] if len(state.new_terms_history) >= 3 else state.new_terms_history - avg_gain = np.mean(recent_gains) - max_possible_gain = 50 # Assumed max new terms per crawl - gain_rate = 1 - (avg_gain / max_possible_gain) - else: - gain_rate = 0.0 - - # Calculate quality consistency - if len(state.quality_scores) > 1: - quality_std = np.std(state.quality_scores[-5:]) - quality_consistency = max(0.0, 1 - quality_std) - else: - quality_consistency = 0.0 - - # Calculate relevance consistency - relevance_consistency = 0.0 - if state.query and len(state.relevance_scores) > 1: - relevance_std = np.std(state.relevance_scores[-5:]) - relevance_consistency = max(0.0, 1 - relevance_std) - - # Combined confidence score - confidence = ( - gain_rate * 0.4 + - quality_consistency * 0.3 + - relevance_consistency * 0.3 - ) - - state.metrics.update({ - 'confidence': confidence, - 'gain_rate': gain_rate, - 'quality_consistency': quality_consistency, - 'relevance_consistency': relevance_consistency, - 'total_terms': len(state.term_frequencies), - 'unique_documents': state.total_documents - }) - - -class AdaptiveCrawler: - """ - Adaptive crawler that learns and improves extraction over time. - - This crawler uses adaptive strategies to determine when sufficient - information has been gathered and which URLs to prioritize. - """ - - def __init__( - self, - scraping_service, - config: AdaptiveConfig, - state_path: Optional[str] = None - ): - """ - Initialize adaptive crawler. - - Args: - scraping_service: The scraping service to use - config: Adaptive crawling configuration - state_path: Optional path to save/load crawling state - """ - self.scraping_service = scraping_service - self.config = config - self.state_path = state_path or config.state_path - - # Initialize strategy - if config.strategy == "statistical": - self.strategy = StatisticalStrategy(config) - else: - raise ValueError(f"Unknown strategy: {config.strategy}") - - # Load or create state - if self.state_path and Path(self.state_path).exists(): - self.state = CrawlState.load(self.state_path) - logger.info("adaptive_state_loaded", path=self.state_path) - else: - self.state = CrawlState() - self.state.confidence_threshold = config.confidence_threshold - self.state.max_crawl_depth = config.max_depth - self.state.max_pages = config.max_pages - self.state.learning_rate = config.learning_rate - - async def adaptive_crawl( - self, - start_url: str, - query: str, - max_links_to_follow: int = 10, - **kwargs - ) -> Dict[str, Any]: - """ - Perform adaptive crawling starting from a URL. - - Args: - start_url: Starting URL for crawling - query: Query to guide crawling relevance - max_links_to_follow: Maximum number of links to follow - **kwargs: Additional crawling parameters - - Returns: - Dictionary with crawling results and metadata - """ - self.state.query = query - crawled_results = [] - - try: - # Initial crawl - logger.info("adaptive_crawl_started", start_url=start_url, query=query) - - if start_url not in self.state.crawled_urls: - initial_result = await self._crawl_single_url(start_url, **kwargs) - if initial_result: - crawled_results.append(initial_result) - await self.strategy.update_state(self.state, initial_result) - self.state.crawled_urls.add(start_url) - - # Extract and score links from initial page - if crawled_results and crawled_results[0].links: - await self._update_pending_links(crawled_results[0].links, start_url) - - # Adaptive crawling loop - crawl_count = 1 - while (await self.strategy.should_continue_crawling(self.state) and - crawl_count < max_links_to_follow and - self.state.pending_links): - - # Get next best URL to crawl - next_url = await self._get_next_url_to_crawl() - if not next_url: - break - - # Crawl the URL - result = await self._crawl_single_url(next_url, **kwargs) - if result: - crawled_results.append(result) - await self.strategy.update_state(self.state, result) - self.state.crawled_urls.add(next_url) - - # Extract new links - if result.links: - await self._update_pending_links(result.links, next_url) - - crawl_count += 1 - - # Save state periodically - if self.config.save_state and self.state_path: - self.state.save(self.state_path) - - # Final state save - if self.config.save_state and self.state_path: - self.state.save(self.state_path) - - logger.info( - "adaptive_crawl_completed", - urls_crawled=len(crawled_results), - confidence=self.state.metrics.get('confidence', 0), - total_terms=len(self.state.term_frequencies) - ) - - return { - 'results': crawled_results, - 'crawl_state': { - 'urls_crawled': len(self.state.crawled_urls), - 'confidence': self.state.metrics.get('confidence', 0), - 'total_terms': len(self.state.term_frequencies), - 'total_documents': self.state.total_documents, - 'avg_quality': np.mean(self.state.quality_scores) if self.state.quality_scores else 0, - 'pending_links': len(self.state.pending_links) - }, - 'metrics': self.state.metrics, - 'query': query - } - - except Exception as e: - logger.error("adaptive_crawl_failed", error=str(e), start_url=start_url) - raise - - async def _crawl_single_url(self, url: str, **kwargs) -> Optional[ScrapedContent]: - """Crawl a single URL using the scraping service.""" - try: - results = await self.scraping_service.scrape_urls([url], **kwargs) - return results[0] if results else None - except Exception as e: - logger.error("single_url_crawl_failed", url=url, error=str(e)) - return None - - async def _update_pending_links(self, links: List[str], base_url: str) -> None: - """Update pending links with relevance scoring.""" - new_links = [] - - for link in links: - # Make absolute URL - abs_link = urljoin(base_url, link) - - # Skip if already crawled or pending - if (abs_link in self.state.crawled_urls or - any(pl['url'] == abs_link for pl in self.state.pending_links)): - continue - - # Score the URL - score = await self.strategy.score_url(abs_link, self.state) - - if score >= self.config.min_relevance_score: - new_links.append({ - 'url': abs_link, - 'score': score, - 'base_url': base_url, - 'discovered_at': len(self.state.crawl_order) - }) - - # Add new links and sort by score - self.state.pending_links.extend(new_links) - self.state.pending_links.sort(key=lambda x: x['score'], reverse=True) - - # Limit pending links to prevent memory issues - if len(self.state.pending_links) > 100: - self.state.pending_links = self.state.pending_links[:100] - - async def _get_next_url_to_crawl(self) -> Optional[str]: - """Get the next best URL to crawl based on scoring.""" - if not self.state.pending_links: - return None - - # Get highest scored link - next_link = self.state.pending_links.pop(0) - return next_link['url'] - - def get_learning_summary(self) -> Dict[str, Any]: - """Get summary of learning progress.""" - if not self.state.term_frequencies: - return {'status': 'no_learning_data'} - - # Top terms - top_terms = sorted( - self.state.term_frequencies.items(), - key=lambda x: x[1], - reverse=True - )[:20] - - return { - 'total_documents': self.state.total_documents, - 'unique_terms': len(self.state.term_frequencies), - 'confidence': self.state.metrics.get('confidence', 0), - 'top_terms': top_terms, - 'avg_quality': np.mean(self.state.quality_scores) if self.state.quality_scores else 0, - 'crawl_efficiency': len(self.state.crawled_urls) / max(1, len(self.state.crawl_order)), - 'information_saturation': np.mean(self.state.new_terms_history[-5:]) if len(self.state.new_terms_history) >= 5 else 0 - } - - -# Convenience functions -async def create_adaptive_crawler( - scraping_service, - confidence_threshold: float = 0.7, - max_depth: int = 5, - max_pages: int = 20, - strategy: str = "statistical", - state_path: Optional[str] = None -) -> AdaptiveCrawler: - """Create an adaptive crawler with specified configuration.""" - config = AdaptiveConfig( - confidence_threshold=confidence_threshold, - max_depth=max_depth, - max_pages=max_pages, - strategy=strategy, - state_path=state_path - ) - - return AdaptiveCrawler( - scraping_service=scraping_service, - config=config, - state_path=state_path - ) diff --git a/apps/backend/app/services/ai_extraction.py b/apps/backend/app/services/ai_extraction.py deleted file mode 100644 index 0c6bafa..0000000 --- a/apps/backend/app/services/ai_extraction.py +++ /dev/null @@ -1,747 +0,0 @@ -""" -AI-powered data extraction service inspired by Firecrawl's Extract feature. - -Provides comprehensive AI extraction capabilities: -- Natural language prompt-based extraction -- JSON schema-driven structured extraction -- Multi-URL and domain-wide extraction -- Web search augmentation -- Source mapping and citation -- Advanced agent configuration -""" - -import asyncio -import time -import json -import uuid -from typing import Dict, List, Optional, Any, Union, Literal -from dataclasses import dataclass, field -from datetime import datetime -from enum import Enum -import structlog - -from app.config import get_settings -from app.models.responses import ScrapedContent -from app.services.enhanced_scraping import get_enhanced_scraping_service -from app.services.multi_search import get_multi_search_service, SearchOptions -from app.services.website_mapping import get_website_mapper, MapOptions, MapStrategy -from app.services.llm_configuration import get_llm_config_service - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class AgentModel(Enum): - """Available LLM models for extraction.""" - GPT_4 = "gpt-4" - GPT_4_TURBO = "gpt-4-turbo" - GPT_3_5_TURBO = "gpt-3.5-turbo" - CLAUDE_3_SONNET = "claude-3-sonnet-20240229" - CLAUDE_3_HAIKU = "claude-3-haiku-20240307" - - -class ExtractionStatus(Enum): - """Status of extraction operation.""" - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - CANCELLED = "cancelled" - - -@dataclass -class AgentConfig: - """Configuration for extraction agent.""" - model: AgentModel = AgentModel.GPT_4 - temperature: float = 0.1 - max_tokens: Optional[int] = None - session_id: Optional[str] = None - custom_instructions: Optional[str] = None - - -@dataclass -class ExtractionSource: - """Source information for extracted data.""" - url: str - title: Optional[str] = None - scraped_at: datetime = field(default_factory=datetime.utcnow) - confidence_score: float = 1.0 - extraction_method: str = "llm" - - -@dataclass -class ExtractedData: - """Container for extracted structured data.""" - data: Any - sources: List[ExtractionSource] = field(default_factory=list) - confidence_score: float = 1.0 - extraction_metadata: Dict[str, Any] = field(default_factory=dict) - - -@dataclass -class ExtractionRequest: - """Request for AI-powered extraction.""" - # Input URLs (optional if using web search) - urls: Optional[List[str]] = None - - # Extraction configuration - prompt: Optional[str] = None - schema: Optional[Dict[str, Any]] = None - system_prompt: Optional[str] = None - - # Search and discovery options - enable_web_search: bool = False - search_query: Optional[str] = None - allow_external_links: bool = False - include_subdomains: bool = True - - # Processing options - show_sources: bool = True - ignore_invalid_urls: bool = True - max_urls: int = 100 - - # Agent configuration - agent: Optional[AgentConfig] = None - - # Advanced options - integration: Optional[str] = None - webhook_url: Optional[str] = None - timeout: int = 300 # 5 minutes default - - -@dataclass -class ExtractionJob: - """Represents an extraction job.""" - id: str - request: ExtractionRequest - status: ExtractionStatus = ExtractionStatus.PENDING - created_at: datetime = field(default_factory=datetime.utcnow) - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None - progress: float = 0.0 - total_urls: int = 0 - processed_urls: int = 0 - error: Optional[str] = None - result: Optional[ExtractedData] = None - - -@dataclass -class ExtractionResponse: - """Response from extraction operation.""" - success: bool - job_id: str - status: ExtractionStatus - data: Optional[Any] = None - sources: List[ExtractionSource] = field(default_factory=list) - processing_time_ms: int = 0 - error: Optional[str] = None - metadata: Dict[str, Any] = field(default_factory=dict) - - -class AIExtractor: - """ - AI-powered data extraction service. - - Provides comprehensive extraction capabilities including: - - Natural language prompt-based extraction - - JSON schema-driven structured extraction - - Multi-URL and domain-wide extraction - - Web search augmentation - - Source mapping and citation - """ - - def __init__(self): - """Initialize AI extractor.""" - self.jobs: Dict[str, ExtractionJob] = {} - self.extraction_stats = { - "total_extractions": 0, - "successful_extractions": 0, - "data_points_extracted": 0, - "urls_processed": 0 - } - - async def extract(self, request: ExtractionRequest) -> ExtractionResponse: - """ - Start an extraction operation and wait for completion. - - Args: - request: Extraction request configuration - - Returns: - ExtractionResponse with extracted data - """ - # Start async extraction - job_id = await self.start_extraction(request) - - # Wait for completion - return await self.wait_for_extraction(job_id, poll_interval=2, timeout=request.timeout) - - async def start_extraction(self, request: ExtractionRequest) -> str: - """ - Start an asynchronous extraction operation. - - Args: - request: Extraction request configuration - - Returns: - Job ID for monitoring - """ - job_id = str(uuid.uuid4()) - job = ExtractionJob(id=job_id, request=request) - - self.jobs[job_id] = job - self.extraction_stats["total_extractions"] += 1 - - logger.info("extraction_job_started", - job_id=job_id, - urls_count=len(request.urls or []), - has_prompt=bool(request.prompt), - has_schema=bool(request.schema)) - - # Start background task - asyncio.create_task(self._execute_extraction(job)) - - return job_id - - async def get_extraction_status(self, job_id: str) -> ExtractionResponse: - """Get status of extraction job.""" - if job_id not in self.jobs: - return ExtractionResponse( - success=False, - job_id=job_id, - status=ExtractionStatus.FAILED, - error="Job not found" - ) - - job = self.jobs[job_id] - - return ExtractionResponse( - success=job.status == ExtractionStatus.COMPLETED, - job_id=job_id, - status=job.status, - data=job.result.data if job.result else None, - sources=job.result.sources if job.result else [], - processing_time_ms=int((datetime.utcnow() - job.created_at).total_seconds() * 1000), - error=job.error, - metadata={ - "progress": job.progress, - "total_urls": job.total_urls, - "processed_urls": job.processed_urls, - "created_at": job.created_at.isoformat(), - "started_at": job.started_at.isoformat() if job.started_at else None, - "completed_at": job.completed_at.isoformat() if job.completed_at else None - } - ) - - async def wait_for_extraction( - self, - job_id: str, - poll_interval: int = 2, - timeout: Optional[int] = None - ) -> ExtractionResponse: - """ - Wait for extraction to complete. - - Args: - job_id: Job ID to monitor - poll_interval: Seconds between status checks - timeout: Maximum wait time in seconds - - Returns: - Final extraction response - """ - start_time = time.time() - - while True: - response = await self.get_extraction_status(job_id) - - if response.status in [ExtractionStatus.COMPLETED, ExtractionStatus.FAILED, ExtractionStatus.CANCELLED]: - return response - - if timeout and (time.time() - start_time) > timeout: - return ExtractionResponse( - success=False, - job_id=job_id, - status=ExtractionStatus.FAILED, - error=f"Extraction timeout after {timeout} seconds" - ) - - await asyncio.sleep(poll_interval) - - async def cancel_extraction(self, job_id: str) -> bool: - """Cancel an extraction job.""" - if job_id not in self.jobs: - return False - - job = self.jobs[job_id] - if job.status in [ExtractionStatus.COMPLETED, ExtractionStatus.FAILED]: - return False - - job.status = ExtractionStatus.CANCELLED - logger.info("extraction_job_cancelled", job_id=job_id) - return True - - async def _execute_extraction(self, job: ExtractionJob): - """Execute extraction job in background.""" - try: - job.status = ExtractionStatus.PROCESSING - job.started_at = datetime.utcnow() - - logger.info("extraction_job_processing", job_id=job.id) - - # Step 1: Discover URLs - urls = await self._discover_urls(job) - job.total_urls = len(urls) - job.progress = 0.1 - - if job.status == ExtractionStatus.CANCELLED: - return - - # Step 2: Scrape URLs - scraped_data = await self._scrape_urls(job, urls) - job.progress = 0.6 - - if job.status == ExtractionStatus.CANCELLED: - return - - # Step 3: Extract structured data - extracted_data = await self._extract_structured_data(job, scraped_data) - job.progress = 0.9 - - if job.status == ExtractionStatus.CANCELLED: - return - - # Step 4: Post-process and finalize - job.result = await self._finalize_extraction(job, extracted_data) - job.status = ExtractionStatus.COMPLETED - job.completed_at = datetime.utcnow() - job.progress = 1.0 - - # Update stats - self.extraction_stats["successful_extractions"] += 1 - self.extraction_stats["urls_processed"] += len(urls) - if job.result: - self.extraction_stats["data_points_extracted"] += self._count_data_points(job.result.data) - - logger.info("extraction_job_completed", - job_id=job.id, - urls_processed=len(urls), - processing_time_ms=int((job.completed_at - job.started_at).total_seconds() * 1000)) - - # Send webhook notification if configured - if job.request.webhook_url: - await self._send_webhook_notification(job) - - except Exception as e: - job.status = ExtractionStatus.FAILED - job.error = str(e) - job.completed_at = datetime.utcnow() - - logger.error("extraction_job_failed", - job_id=job.id, - error=str(e)) - - async def _discover_urls(self, job: ExtractionJob) -> List[str]: - """Discover URLs for extraction.""" - request = job.request - urls = [] - - # Use provided URLs - if request.urls: - urls.extend(request.urls) - - # Use web search if enabled - if request.enable_web_search and request.search_query: - try: - search_service = await get_multi_search_service() - search_options = SearchOptions( - query=request.search_query, - num_results=min(request.max_urls, 50), - lang="en", - country="us" - ) - - search_results = await search_service.search(search_options) - search_urls = [result.url for result in search_results] - urls.extend(search_urls) - - logger.info("web_search_urls_discovered", - job_id=job.id, - search_urls=len(search_urls)) - - except Exception as e: - logger.warning("web_search_failed", job_id=job.id, error=str(e)) - - # Discover additional URLs from domains if wildcards used - domain_urls = [] - for url in urls[:]: # Copy to avoid modification during iteration - if url.endswith('/*'): - try: - base_url = url[:-2] # Remove /* - mapper = await get_website_mapper() - - map_options = MapOptions( - strategy=MapStrategy.COMBINED, - limit=min(request.max_urls, 100), - include_subdomains=request.include_subdomains, - allow_external_links=request.allow_external_links - ) - - mapping_result = await mapper.map_website(base_url, map_options) - if mapping_result.success: - discovered = [du.url for du in mapping_result.discovered_urls] - domain_urls.extend(discovered) - - logger.info("domain_urls_discovered", - job_id=job.id, - base_url=base_url, - discovered_urls=len(discovered)) - - except Exception as e: - logger.warning("domain_discovery_failed", - job_id=job.id, - url=url, - error=str(e)) - - urls.extend(domain_urls) - - # Remove duplicates and filter invalid URLs - unique_urls = list(set(urls)) - - if request.ignore_invalid_urls: - valid_urls = [] - for url in unique_urls: - if self._is_valid_url(url): - valid_urls.append(url) - unique_urls = valid_urls - - # Apply limit - if len(unique_urls) > request.max_urls: - unique_urls = unique_urls[:request.max_urls] - - logger.info("urls_discovered", - job_id=job.id, - total_urls=len(unique_urls)) - - return unique_urls - - async def _scrape_urls(self, job: ExtractionJob, urls: List[str]) -> List[ScrapedContent]: - """Scrape content from URLs.""" - try: - scraping_service = await get_enhanced_scraping_service() - - # Scrape in batches to avoid overwhelming - batch_size = 10 - all_scraped = [] - - for i in range(0, len(urls), batch_size): - if job.status == ExtractionStatus.CANCELLED: - break - - batch_urls = urls[i:i + batch_size] - - try: - batch_results = await scraping_service.scrape_urls_enhanced(batch_urls) - successful_results = [r for r in batch_results if r.extraction_success] - all_scraped.extend(successful_results) - - job.processed_urls += len(batch_results) - job.progress = 0.1 + (0.5 * job.processed_urls / job.total_urls) - - logger.debug("batch_scraped", - job_id=job.id, - batch_size=len(batch_urls), - successful=len(successful_results)) - - except Exception as e: - logger.warning("batch_scraping_failed", - job_id=job.id, - batch_urls=batch_urls, - error=str(e)) - - # Small delay between batches - await asyncio.sleep(0.5) - - logger.info("scraping_completed", - job_id=job.id, - total_scraped=len(all_scraped), - total_attempted=len(urls)) - - return all_scraped - - except Exception as e: - logger.error("scraping_failed", job_id=job.id, error=str(e)) - raise e - - async def _extract_structured_data( - self, - job: ExtractionJob, - scraped_data: List[ScrapedContent] - ) -> ExtractedData: - """Extract structured data using LLM.""" - request = job.request - - if not request.prompt and not request.schema: - raise ValueError("Either prompt or schema is required for extraction") - - try: - # Combine all scraped content - combined_content = [] - sources = [] - - for scraped in scraped_data: - content_parts = [] - if scraped.title: - content_parts.append(f"Title: {scraped.title}") - if scraped.text: - content_parts.append(f"Content: {scraped.text[:2000]}") # Limit content length - - if content_parts: - combined_content.append(f"URL: {scraped.url}\n" + "\n".join(content_parts)) - - sources.append(ExtractionSource( - url=scraped.url, - title=scraped.title, - confidence_score=scraped.content_quality_score or 1.0, - extraction_method="scraping" - )) - - # Prepare LLM prompt - system_prompt = request.system_prompt or """You are an expert data extraction assistant. -Extract structured information from web content according to the provided prompt or schema. -Return only valid JSON without any additional text or formatting.""" - - if request.schema: - if request.prompt: - user_prompt = f"""Extract data according to this prompt: {request.prompt} - -Use this JSON schema as the structure: -{json.dumps(request.schema, indent=2)} - -Content to extract from: -{chr(10).join(combined_content[:10])}""" # Limit to first 10 sources - else: - user_prompt = f"""Extract data according to this JSON schema: -{json.dumps(request.schema, indent=2)} - -Content to extract from: -{chr(10).join(combined_content[:10])}""" - else: - user_prompt = f"""Extract data according to this prompt: {request.prompt} - -Return the result as valid JSON. - -Content to extract from: -{chr(10).join(combined_content[:10])}""" - - # Call LLM service - llm_service = await get_llm_config_service() - agent_config = request.agent or AgentConfig() - - extraction_result = await llm_service.generate_response( - prompt=user_prompt, - system_prompt=system_prompt, - model=agent_config.model.value, - temperature=agent_config.temperature, - max_tokens=agent_config.max_tokens - ) - - # Parse LLM response as JSON - try: - extracted_json = json.loads(extraction_result) - except json.JSONDecodeError: - # Try to extract JSON from response if it's wrapped in text - import re - json_match = re.search(r'\{.*\}', extraction_result, re.DOTALL) - if json_match: - extracted_json = json.loads(json_match.group()) - else: - extracted_json = {"extracted_text": extraction_result} - - return ExtractedData( - data=extracted_json, - sources=sources, - confidence_score=0.9, # High confidence for LLM extraction - extraction_metadata={ - "model_used": agent_config.model.value, - "sources_count": len(sources), - "extraction_method": "llm_structured" - } - ) - - except Exception as e: - logger.error("structured_extraction_failed", job_id=job.id, error=str(e)) - - # Fallback to simple text extraction - return ExtractedData( - data={"error": str(e), "fallback_content": [s.text[:500] for s in scraped_data[:5]]}, - sources=sources, - confidence_score=0.3, - extraction_metadata={"extraction_method": "fallback"} - ) - - async def _finalize_extraction(self, job: ExtractionJob, extracted_data: ExtractedData) -> ExtractedData: - """Finalize extraction with post-processing.""" - request = job.request - - # Add job metadata - extracted_data.extraction_metadata.update({ - "job_id": job.id, - "processing_time_ms": int((datetime.utcnow() - job.started_at).total_seconds() * 1000), - "urls_processed": job.processed_urls, - "integration": request.integration - }) - - # Filter sources if not requested - if not request.show_sources: - extracted_data.sources = [] - - return extracted_data - - def _is_valid_url(self, url: str) -> bool: - """Check if URL is valid.""" - try: - from urllib.parse import urlparse - result = urlparse(url) - return all([result.scheme, result.netloc]) - except Exception: - return False - - def _count_data_points(self, data: Any) -> int: - """Count data points in extracted data.""" - if isinstance(data, dict): - return sum(self._count_data_points(v) for v in data.values()) - elif isinstance(data, list): - return sum(self._count_data_points(item) for item in data) - else: - return 1 - - async def _send_webhook_notification(self, job: ExtractionJob): - """Send webhook notification about job completion.""" - try: - import httpx - - webhook_data = { - "job_id": job.id, - "status": job.status.value, - "completed_at": job.completed_at.isoformat() if job.completed_at else None, - "urls_processed": job.processed_urls, - "success": job.status == ExtractionStatus.COMPLETED, - "error": job.error - } - - async with httpx.AsyncClient() as client: - response = await client.post( - job.request.webhook_url, - json=webhook_data, - timeout=10 - ) - - if response.status_code == 200: - logger.info("webhook_notification_sent", - job_id=job.id, - webhook_url=job.request.webhook_url) - else: - logger.warning("webhook_notification_failed", - job_id=job.id, - status_code=response.status_code) - - except Exception as e: - logger.error("webhook_notification_error", - job_id=job.id, - error=str(e)) - - async def get_extraction_stats(self) -> Dict[str, Any]: - """Get extraction service statistics.""" - active_jobs = len([j for j in self.jobs.values() if j.status == ExtractionStatus.PROCESSING]) - completed_jobs = len([j for j in self.jobs.values() if j.status == ExtractionStatus.COMPLETED]) - - return { - "extraction_stats": self.extraction_stats, - "active_jobs": active_jobs, - "completed_jobs": completed_jobs, - "total_jobs": len(self.jobs), - "success_rate": ( - self.extraction_stats["successful_extractions"] / - max(1, self.extraction_stats["total_extractions"]) - ) if self.extraction_stats["total_extractions"] > 0 else 0, - "avg_urls_per_extraction": ( - self.extraction_stats["urls_processed"] / - max(1, self.extraction_stats["successful_extractions"]) - ) if self.extraction_stats["successful_extractions"] > 0 else 0 - } - - -# Singleton service -_ai_extractor: Optional[AIExtractor] = None - - -async def get_ai_extractor() -> AIExtractor: - """Get or create AI extractor service instance.""" - global _ai_extractor - - if _ai_extractor is None: - _ai_extractor = AIExtractor() - - return _ai_extractor - - -# Convenience functions -async def extract_with_prompt( - urls: List[str], - prompt: str, - enable_web_search: bool = False, - show_sources: bool = True -) -> ExtractionResponse: - """ - Extract data using natural language prompt. - - Args: - urls: URLs to extract from - prompt: Natural language extraction prompt - enable_web_search: Enable web search augmentation - show_sources: Include source information - - Returns: - ExtractionResponse with extracted data - """ - extractor = await get_ai_extractor() - - request = ExtractionRequest( - urls=urls, - prompt=prompt, - enable_web_search=enable_web_search, - show_sources=show_sources - ) - - return await extractor.extract(request) - - -async def extract_with_schema( - urls: List[str], - schema: Dict[str, Any], - system_prompt: Optional[str] = None, - show_sources: bool = True -) -> ExtractionResponse: - """ - Extract data using JSON schema. - - Args: - urls: URLs to extract from - schema: JSON schema for structured extraction - system_prompt: Optional system prompt - show_sources: Include source information - - Returns: - ExtractionResponse with extracted data - """ - extractor = await get_ai_extractor() - - request = ExtractionRequest( - urls=urls, - schema=schema, - system_prompt=system_prompt, - show_sources=show_sources - ) - - return await extractor.extract(request) diff --git a/apps/backend/app/services/attributes_extraction.py b/apps/backend/app/services/attributes_extraction.py deleted file mode 100644 index 7d2ec44..0000000 --- a/apps/backend/app/services/attributes_extraction.py +++ /dev/null @@ -1,700 +0,0 @@ -""" -HTML attributes extraction service inspired by Firecrawl. - -Provides comprehensive attribute extraction capabilities: -- CSS selector-based attribute extraction -- Bulk attribute extraction -- Multi-element attribute collection -- Advanced filtering and processing -- Structured attribute analysis -""" - -import asyncio -import time -import re -from typing import Dict, List, Optional, Any, Union, Set, Callable -from dataclasses import dataclass, field -from enum import Enum -from urllib.parse import urljoin, urlparse -import structlog - -from app.config import get_settings -from app.services.enhanced_scraping import get_enhanced_scraping_service -from app.utils.text_processing import sanitize_text, clean_text - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class AttributeProcessingType(Enum): - """Types of attribute processing.""" - RAW = "raw" - CLEANED = "cleaned" - URLS_RESOLVED = "urls_resolved" - NUMERIC = "numeric" - BOOLEAN = "boolean" - LIST = "list" - - -@dataclass -class AttributeExtractionRule: - """Rule for extracting attributes.""" - selector: str - attribute: str - processing: AttributeProcessingType = AttributeProcessingType.RAW - filter_empty: bool = True - filter_duplicates: bool = True - limit: Optional[int] = None - transform: Optional[str] = None # CSS transform function or regex - validation_pattern: Optional[str] = None # Regex for validation - - -@dataclass -class AttributeResult: - """Result of attribute extraction for a single element.""" - element_index: int - selector_match: str - attribute_name: str - raw_value: str - processed_value: Any - element_text: Optional[str] = None - element_tag: Optional[str] = None - element_classes: List[str] = field(default_factory=list) - element_id: Optional[str] = None - - -@dataclass -class SelectorAttributeResults: - """Results for a specific selector and attribute combination.""" - selector: str - attribute: str - values: List[str] - processed_values: List[Any] - element_count: int - results: List[AttributeResult] = field(default_factory=list) - - -@dataclass -class AttributesExtractionConfig: - """Configuration for attributes extraction.""" - rules: List[AttributeExtractionRule] = field(default_factory=list) - base_url: Optional[str] = None # For URL resolution - include_element_context: bool = True - max_elements_per_selector: int = 1000 - timeout: int = 30 - resolve_relative_urls: bool = True - normalize_whitespace: bool = True - - # Advanced options - extract_computed_styles: bool = False - include_xpath_location: bool = False - extract_surrounding_context: bool = False - context_window: int = 50 # Characters around element - - -@dataclass -class AttributesExtractionResult: - """Complete result of attributes extraction.""" - url: str - extractions: List[SelectorAttributeResults] - total_attributes_extracted: int - total_elements_processed: int - processing_time_ms: int - success: bool - error: Optional[str] = None - metadata: Dict[str, Any] = field(default_factory=dict) - - -class AttributeProcessor: - """Handles processing of extracted attribute values.""" - - def __init__(self, base_url: Optional[str] = None): - self.base_url = base_url - - def process_value( - self, - value: str, - processing_type: AttributeProcessingType, - transform: Optional[str] = None - ) -> Any: - """Process attribute value according to type.""" - if not value and processing_type != AttributeProcessingType.BOOLEAN: - return value - - try: - if processing_type == AttributeProcessingType.RAW: - return value - - elif processing_type == AttributeProcessingType.CLEANED: - cleaned = clean_text(value) if value else "" - return cleaned.strip() - - elif processing_type == AttributeProcessingType.URLS_RESOLVED: - if self.base_url and value: - return urljoin(self.base_url, value) - return value - - elif processing_type == AttributeProcessingType.NUMERIC: - if not value: - return None - # Extract first number from string - number_match = re.search(r'-?\d+(?:\.\d+)?', value) - if number_match: - num_str = number_match.group() - return float(num_str) if '.' in num_str else int(num_str) - return None - - elif processing_type == AttributeProcessingType.BOOLEAN: - if not value: - return False - return value.lower() in ['true', '1', 'yes', 'on', 'checked', 'selected'] - - elif processing_type == AttributeProcessingType.LIST: - if not value: - return [] - # Split by common delimiters - delimiters = [',', ';', '|', '\n'] - items = [value] - for delimiter in delimiters: - new_items = [] - for item in items: - new_items.extend([x.strip() for x in item.split(delimiter)]) - items = new_items - return [item for item in items if item] - - # Apply custom transform if provided - if transform: - return self._apply_transform(value, transform) - - return value - - except Exception as e: - logger.warning("attribute_processing_failed", - value=value, - processing_type=processing_type.value, - error=str(e)) - return value - - def _apply_transform(self, value: str, transform: str) -> Any: - """Apply custom transform to value.""" - try: - # Handle regex transforms - if transform.startswith('regex:'): - pattern = transform[6:] # Remove 'regex:' prefix - match = re.search(pattern, value) - return match.group(1) if match and match.groups() else match.group(0) if match else None - - # Handle JavaScript-like transforms - elif transform.startswith('js:'): - # Simplified JS-like transforms - js_code = transform[3:] - if 'toLowerCase()' in js_code: - return value.lower() - elif 'toUpperCase()' in js_code: - return value.upper() - elif 'trim()' in js_code: - return value.strip() - - # Handle CSS-like transforms - elif transform == 'uppercase': - return value.upper() - elif transform == 'lowercase': - return value.lower() - elif transform == 'capitalize': - return value.capitalize() - - return value - - except Exception as e: - logger.warning("transform_failed", value=value, transform=transform, error=str(e)) - return value - - def validate_value(self, value: Any, pattern: Optional[str]) -> bool: - """Validate processed value against pattern.""" - if not pattern: - return True - - try: - str_value = str(value) if value is not None else "" - return bool(re.search(pattern, str_value)) - except Exception: - return False - - -class AttributesExtractor: - """ - HTML attributes extraction service. - - Provides comprehensive attribute extraction including: - - CSS selector-based extraction - - Multiple processing types - - Bulk extraction operations - - Advanced filtering and validation - """ - - def __init__(self): - """Initialize attributes extractor.""" - self.extraction_stats = { - "total_extractions": 0, - "successful_extractions": 0, - "attributes_extracted": 0, - "elements_processed": 0 - } - - async def extract_attributes( - self, - url: str, - config: AttributesExtractionConfig - ) -> AttributesExtractionResult: - """ - Extract attributes from a web page. - - Args: - url: URL to extract attributes from - config: Extraction configuration - - Returns: - AttributesExtractionResult with extracted attributes - """ - start_time = time.time() - - self.extraction_stats["total_extractions"] += 1 - - logger.info("attributes_extraction_started", - url=url, - rules_count=len(config.rules)) - - try: - # Scrape the page - scraping_service = await get_enhanced_scraping_service() - scrape_results = await scraping_service.scrape_urls_enhanced([url]) - - if not scrape_results or not scrape_results[0].extraction_success: - raise ValueError("Failed to scrape page content") - - scraped_content = scrape_results[0] - html_content = scraped_content.html - - if not html_content: - raise ValueError("No HTML content available") - - # Parse HTML - from bs4 import BeautifulSoup - soup = BeautifulSoup(html_content, 'lxml') - - # Set base URL for URL resolution - base_url = config.base_url or url - processor = AttributeProcessor(base_url) - - # Extract attributes for each rule - extractions = [] - total_attributes = 0 - total_elements = 0 - - for rule in config.rules: - try: - logger.debug("processing_extraction_rule", - selector=rule.selector, - attribute=rule.attribute) - - extraction_result = await self._extract_for_rule( - soup, rule, processor, config - ) - - extractions.append(extraction_result) - total_attributes += len(extraction_result.values) - total_elements += extraction_result.element_count - - except Exception as e: - logger.error("rule_extraction_failed", - selector=rule.selector, - attribute=rule.attribute, - error=str(e)) - - # Add empty result for failed rule - extractions.append(SelectorAttributeResults( - selector=rule.selector, - attribute=rule.attribute, - values=[], - processed_values=[], - element_count=0 - )) - - processing_time_ms = int((time.time() - start_time) * 1000) - - # Update stats - self.extraction_stats["successful_extractions"] += 1 - self.extraction_stats["attributes_extracted"] += total_attributes - self.extraction_stats["elements_processed"] += total_elements - - result = AttributesExtractionResult( - url=url, - extractions=extractions, - total_attributes_extracted=total_attributes, - total_elements_processed=total_elements, - processing_time_ms=processing_time_ms, - success=True, - metadata={ - "page_title": scraped_content.title, - "rules_processed": len(config.rules), - "html_size": len(html_content) - } - ) - - logger.info("attributes_extraction_completed", - url=url, - total_attributes=total_attributes, - total_elements=total_elements, - processing_time_ms=processing_time_ms) - - return result - - except Exception as e: - processing_time_ms = int((time.time() - start_time) * 1000) - error_msg = str(e) - - logger.error("attributes_extraction_failed", - url=url, - error=error_msg, - processing_time_ms=processing_time_ms) - - return AttributesExtractionResult( - url=url, - extractions=[], - total_attributes_extracted=0, - total_elements_processed=0, - processing_time_ms=processing_time_ms, - success=False, - error=error_msg - ) - - async def _extract_for_rule( - self, - soup: Any, - rule: AttributeExtractionRule, - processor: AttributeProcessor, - config: AttributesExtractionConfig - ) -> SelectorAttributeResults: - """Extract attributes for a specific rule.""" - try: - # Find elements matching selector - elements = soup.select(rule.selector) - - if rule.limit: - elements = elements[:rule.limit] - elif len(elements) > config.max_elements_per_selector: - elements = elements[:config.max_elements_per_selector] - - results = [] - values = [] - processed_values = [] - - for i, element in enumerate(elements): - try: - # Get attribute value - raw_value = element.get(rule.attribute, "") - - if not raw_value and rule.filter_empty: - continue - - # Process value - processed_value = processor.process_value( - raw_value, rule.processing, rule.transform - ) - - # Validate if pattern provided - if rule.validation_pattern and not processor.validate_value( - processed_value, rule.validation_pattern - ): - continue - - # Handle URL normalization - if config.normalize_whitespace and isinstance(processed_value, str): - processed_value = re.sub(r'\s+', ' ', processed_value).strip() - - # Create result - attribute_result = AttributeResult( - element_index=i, - selector_match=rule.selector, - attribute_name=rule.attribute, - raw_value=raw_value, - processed_value=processed_value - ) - - # Add element context if requested - if config.include_element_context: - attribute_result.element_text = element.get_text(strip=True)[:200] # Limit text - attribute_result.element_tag = element.name - attribute_result.element_classes = element.get('class', []) - attribute_result.element_id = element.get('id') - - results.append(attribute_result) - values.append(raw_value) - processed_values.append(processed_value) - - except Exception as e: - logger.warning("element_processing_failed", - selector=rule.selector, - element_index=i, - error=str(e)) - continue - - # Remove duplicates if requested - if rule.filter_duplicates: - unique_results = [] - seen_values = set() - filtered_values = [] - filtered_processed = [] - - for result, value, processed in zip(results, values, processed_values): - value_key = str(processed) if processed is not None else str(value) - - if value_key not in seen_values: - seen_values.add(value_key) - unique_results.append(result) - filtered_values.append(value) - filtered_processed.append(processed) - - results = unique_results - values = filtered_values - processed_values = filtered_processed - - return SelectorAttributeResults( - selector=rule.selector, - attribute=rule.attribute, - values=values, - processed_values=processed_values, - element_count=len(results), - results=results - ) - - except Exception as e: - logger.error("rule_extraction_failed", - selector=rule.selector, - attribute=rule.attribute, - error=str(e)) - - return SelectorAttributeResults( - selector=rule.selector, - attribute=rule.attribute, - values=[], - processed_values=[], - element_count=0 - ) - - async def extract_bulk_attributes( - self, - urls: List[str], - config: AttributesExtractionConfig, - max_concurrent: int = 5 - ) -> List[AttributesExtractionResult]: - """Extract attributes from multiple URLs concurrently.""" - semaphore = asyncio.Semaphore(max_concurrent) - - async def extract_single(url: str) -> AttributesExtractionResult: - async with semaphore: - return await self.extract_attributes(url, config) - - logger.info("bulk_attributes_extraction_started", urls_count=len(urls)) - - results = await asyncio.gather( - *[extract_single(url) for url in urls], - return_exceptions=True - ) - - # Handle exceptions - final_results = [] - for i, result in enumerate(results): - if isinstance(result, Exception): - logger.error("bulk_extraction_failed", url=urls[i], error=str(result)) - final_results.append(AttributesExtractionResult( - url=urls[i], - extractions=[], - total_attributes_extracted=0, - total_elements_processed=0, - processing_time_ms=0, - success=False, - error=str(result) - )) - else: - final_results.append(result) - - successful_results = [r for r in final_results if r.success] - - logger.info("bulk_attributes_extraction_completed", - urls_count=len(urls), - successful_extractions=len(successful_results), - total_attributes=sum(r.total_attributes_extracted for r in successful_results)) - - return final_results - - async def extract_common_attributes( - self, - url: str, - base_url: Optional[str] = None - ) -> AttributesExtractionResult: - """Extract commonly useful attributes from a page.""" - common_rules = [ - # Links - AttributeExtractionRule("a", "href", AttributeProcessingType.URLS_RESOLVED), - AttributeExtractionRule("a", "title", AttributeProcessingType.CLEANED), - - # Images - AttributeExtractionRule("img", "src", AttributeProcessingType.URLS_RESOLVED), - AttributeExtractionRule("img", "alt", AttributeProcessingType.CLEANED), - AttributeExtractionRule("img", "title", AttributeProcessingType.CLEANED), - - # Forms - AttributeExtractionRule("form", "action", AttributeProcessingType.URLS_RESOLVED), - AttributeExtractionRule("form", "method", AttributeProcessingType.RAW), - AttributeExtractionRule("input", "type", AttributeProcessingType.RAW), - AttributeExtractionRule("input", "name", AttributeProcessingType.RAW), - - # Meta tags - AttributeExtractionRule("meta[name]", "name", AttributeProcessingType.RAW), - AttributeExtractionRule("meta[name]", "content", AttributeProcessingType.CLEANED), - AttributeExtractionRule("meta[property]", "property", AttributeProcessingType.RAW), - AttributeExtractionRule("meta[property]", "content", AttributeProcessingType.CLEANED), - - # Scripts and styles - AttributeExtractionRule("script", "src", AttributeProcessingType.URLS_RESOLVED), - AttributeExtractionRule("link", "href", AttributeProcessingType.URLS_RESOLVED), - AttributeExtractionRule("link", "rel", AttributeProcessingType.RAW), - - # Structured data - AttributeExtractionRule("[itemtype]", "itemtype", AttributeProcessingType.RAW), - AttributeExtractionRule("[itemprop]", "itemprop", AttributeProcessingType.RAW), - - # IDs and classes - AttributeExtractionRule("[id]", "id", AttributeProcessingType.RAW), - AttributeExtractionRule("[class]", "class", AttributeProcessingType.LIST), - ] - - config = AttributesExtractionConfig( - rules=common_rules, - base_url=base_url, - include_element_context=True - ) - - return await self.extract_attributes(url, config) - - async def get_extraction_stats(self) -> Dict[str, Any]: - """Get extraction service statistics.""" - return { - "extraction_stats": self.extraction_stats, - "success_rate": ( - self.extraction_stats["successful_extractions"] / - max(1, self.extraction_stats["total_extractions"]) - ) if self.extraction_stats["total_extractions"] > 0 else 0, - "avg_attributes_per_extraction": ( - self.extraction_stats["attributes_extracted"] / - max(1, self.extraction_stats["successful_extractions"]) - ) if self.extraction_stats["successful_extractions"] > 0 else 0, - "avg_elements_per_extraction": ( - self.extraction_stats["elements_processed"] / - max(1, self.extraction_stats["successful_extractions"]) - ) if self.extraction_stats["successful_extractions"] > 0 else 0 - } - - -# Singleton service -_attributes_extractor: Optional[AttributesExtractor] = None - - -async def get_attributes_extractor() -> AttributesExtractor: - """Get or create attributes extractor service instance.""" - global _attributes_extractor - - if _attributes_extractor is None: - _attributes_extractor = AttributesExtractor() - - return _attributes_extractor - - -# Convenience functions -async def extract_page_attributes( - url: str, - selector_attribute_pairs: List[tuple], # [(selector, attribute), ...] - processing_type: str = "cleaned", - base_url: Optional[str] = None -) -> AttributesExtractionResult: - """ - Convenience function for extracting specific attributes. - - Args: - url: URL to extract from - selector_attribute_pairs: List of (selector, attribute) tuples - processing_type: How to process values - base_url: Base URL for URL resolution - - Returns: - AttributesExtractionResult with extracted attributes - """ - extractor = await get_attributes_extractor() - - rules = [] - processing = AttributeProcessingType(processing_type) - - for selector, attribute in selector_attribute_pairs: - rules.append(AttributeExtractionRule( - selector=selector, - attribute=attribute, - processing=processing - )) - - config = AttributesExtractionConfig( - rules=rules, - base_url=base_url - ) - - return await extractor.extract_attributes(url, config) - - -async def extract_all_links(url: str) -> List[str]: - """Extract all links from a page.""" - extractor = await get_attributes_extractor() - - config = AttributesExtractionConfig( - rules=[AttributeExtractionRule( - "a", "href", AttributeProcessingType.URLS_RESOLVED - )], - base_url=url - ) - - result = await extractor.extract_attributes(url, config) - - if result.success and result.extractions: - return result.extractions[0].processed_values - - return [] - - -async def extract_all_images(url: str) -> List[Dict[str, str]]: - """Extract all images with src and alt attributes.""" - extractor = await get_attributes_extractor() - - config = AttributesExtractionConfig( - rules=[ - AttributeExtractionRule("img", "src", AttributeProcessingType.URLS_RESOLVED), - AttributeExtractionRule("img", "alt", AttributeProcessingType.CLEANED), - ], - base_url=url, - include_element_context=True - ) - - result = await extractor.extract_attributes(url, config) - - if result.success and len(result.extractions) >= 2: - src_results = result.extractions[0] - alt_results = result.extractions[1] - - images = [] - for i in range(min(len(src_results.results), len(alt_results.results))): - images.append({ - "src": src_results.results[i].processed_value, - "alt": alt_results.results[i].processed_value or "" - }) - - return images - - return [] diff --git a/apps/backend/app/services/auth_service.py b/apps/backend/app/services/auth_service.py deleted file mode 100644 index af955bb..0000000 --- a/apps/backend/app/services/auth_service.py +++ /dev/null @@ -1,433 +0,0 @@ -""" -Authentication and user management service. -""" -from typing import Optional, Dict, Any, List -from datetime import datetime, timedelta -import jwt -import secrets -import hashlib -import structlog -from passlib.context import CryptContext - -from app.config import get_settings -from app.models.users import User, UserAPIKey, UsageRecord -from app.services.database import DatabaseService, get_database_service -from fastapi import Depends -from app.utils.exceptions import AuthenticationException, UnauthorizedException - -logger = structlog.get_logger(__name__) -settings = get_settings() - -# Password hashing -pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") - - -class AuthService: - """Service for authentication and user management.""" - - def __init__(self, db_service: DatabaseService): - self.db = db_service - self.secret_key = settings.secret_key - self.algorithm = "HS256" - self.access_token_expire_minutes = 60 * 24 # 24 hours - self.refresh_token_expire_days = 30 - - async def register_user( - self, - email: str, - password: str, - full_name: Optional[str] = None, - company: Optional[str] = None - ) -> User: - """Register a new user.""" - # Check if user exists - existing = await self.db.get_user_by_email(email) - if existing: - raise AuthenticationException("User with this email already exists") - - # Generate salt and hash password - salt = secrets.token_hex(16) - password_hash = pwd_context.hash(password + salt) - - # Create user - user = User( - email=email, - password_hash=password_hash, - salt=salt, - full_name=full_name, - company=company, - verification_token=secrets.token_urlsafe(32) - ) - - user = await self.db.create_user(user) - - # Create initial API key - await self.create_api_key(user, "Default API Key") - - # Initialize usage record for current month - await self._initialize_usage_record(user) - - logger.info("user_registered", user_id=user.id, email=email) - - return user - - async def login(self, email: str, password: str) -> Dict[str, Any]: - """Authenticate user and return tokens.""" - # Get user - user = await self.db.get_user_by_email(email) - if not user: - raise UnauthorizedException("Invalid email or password") - - # Verify password - if not pwd_context.verify(password + user.salt, user.password_hash): - raise UnauthorizedException("Invalid email or password") - - # Check if user is active - if not user.is_active: - raise UnauthorizedException("Account is disabled") - - # Update last login - user.last_login_at = datetime.utcnow() - await self.db.update_user(user) - - # Generate tokens - access_token = self.create_access_token(user) - refresh_token = self.create_refresh_token(user) - - logger.info("user_login", user_id=user.id) - - return { - "access_token": access_token, - "refresh_token": refresh_token, - "token_type": "bearer", - "expires_in": self.access_token_expire_minutes * 60, - "user": { - "id": user.id, - "email": user.email, - "full_name": user.full_name, - "is_verified": user.is_verified, - "plan": user.current_plan.value if user.current_plan else "free" - } - } - - def create_access_token(self, user: User) -> str: - """Create JWT access token.""" - expire = datetime.utcnow() + timedelta(minutes=self.access_token_expire_minutes) - payload = { - "sub": str(user.id), - "email": user.email, - "exp": expire, - "iat": datetime.utcnow(), - "type": "access" - } - return jwt.encode(payload, self.secret_key, algorithm=self.algorithm) - - def create_refresh_token(self, user: User) -> str: - """Create JWT refresh token.""" - expire = datetime.utcnow() + timedelta(days=self.refresh_token_expire_days) - payload = { - "sub": str(user.id), - "exp": expire, - "iat": datetime.utcnow(), - "type": "refresh" - } - return jwt.encode(payload, self.secret_key, algorithm=self.algorithm) - - async def verify_token(self, token: str) -> Optional[User]: - """Verify JWT token and return user.""" - try: - payload = jwt.decode(token, self.secret_key, algorithms=[self.algorithm]) - user_id = payload.get("sub") - - if not user_id: - return None - - user = await self.db.get_user(int(user_id)) - if not user or not user.is_active: - return None - - return user - - except jwt.ExpiredSignatureError: - logger.warning("token_expired") - return None - except jwt.JWTError: - logger.warning("invalid_token") - return None - - async def refresh_tokens(self, refresh_token: str) -> Dict[str, Any]: - """Refresh access token using refresh token.""" - try: - payload = jwt.decode(refresh_token, self.secret_key, algorithms=[self.algorithm]) - - if payload.get("type") != "refresh": - raise UnauthorizedException("Invalid refresh token") - - user_id = payload.get("sub") - if not user_id: - raise UnauthorizedException("Invalid refresh token") - - user = await self.db.get_user(int(user_id)) - if not user or not user.is_active: - raise UnauthorizedException("User not found or inactive") - - # Generate new tokens - new_access_token = self.create_access_token(user) - new_refresh_token = self.create_refresh_token(user) - - return { - "access_token": new_access_token, - "refresh_token": new_refresh_token, - "token_type": "bearer", - "expires_in": self.access_token_expire_minutes * 60 - } - - except jwt.ExpiredSignatureError: - raise UnauthorizedException("Refresh token expired") - except jwt.JWTError: - raise UnauthorizedException("Invalid refresh token") - - async def create_api_key( - self, - user: User, - name: str, - description: Optional[str] = None, - scopes: Optional[List[str]] = None - ) -> UserAPIKey: - """Create an API key for a user.""" - # Generate secure API key - key_prefix = "sk_" - if settings.environment == "production": - key_prefix = "sk_live_" - elif settings.environment == "development": - key_prefix = "sk_test_" - - key = key_prefix + secrets.token_urlsafe(32) - - # Create API key record - api_key = UserAPIKey( - user_id=user.id, - key=key, - name=name, - description=description, - scopes=scopes or ["read", "write"] - ) - - api_key = await self.db.create_api_key(api_key) - - logger.info("api_key_created", user_id=user.id, key_id=api_key.id) - - return api_key - - async def verify_api_key(self, key: str) -> Optional[User]: - """Verify API key and return associated user.""" - api_key = await self.db.get_api_key_by_value(key) - - if not api_key or not api_key.is_active: - return None - - # Check expiration - if api_key.expires_at and api_key.expires_at < datetime.utcnow(): - return None - - # Update last used - api_key.last_used_at = datetime.utcnow() - api_key.request_count += 1 - await self.db.update_api_key(api_key) - - # Get user - user = await self.db.get_user(api_key.user_id) - if not user or not user.is_active: - return None - - return user - - async def reset_password_request(self, email: str) -> str: - """Request password reset.""" - user = await self.db.get_user_by_email(email) - if not user: - # Don't reveal if user exists - return "If the email exists, a reset link has been sent" - - # Generate reset token - reset_token = secrets.token_urlsafe(32) - user.reset_token = reset_token - user.reset_token_expires = datetime.utcnow() + timedelta(hours=1) - await self.db.update_user(user) - - # TODO: Send email with reset link - logger.info("password_reset_requested", user_id=user.id) - - return reset_token - - async def reset_password(self, token: str, new_password: str) -> bool: - """Reset password using token.""" - user = await self.db.get_user_by_reset_token(token) - - if not user: - raise AuthenticationException("Invalid reset token") - - if user.reset_token_expires < datetime.utcnow(): - raise AuthenticationException("Reset token expired") - - # Generate new salt and hash password - salt = secrets.token_hex(16) - password_hash = pwd_context.hash(new_password + salt) - - # Update user - user.password_hash = password_hash - user.salt = salt - user.reset_token = None - user.reset_token_expires = None - await self.db.update_user(user) - - logger.info("password_reset_completed", user_id=user.id) - - return True - - async def verify_email(self, token: str) -> bool: - """Verify email using token.""" - user = await self.db.get_user_by_verification_token(token) - - if not user: - raise AuthenticationException("Invalid verification token") - - user.is_verified = True - user.email_verified_at = datetime.utcnow() - user.verification_token = None - await self.db.update_user(user) - - logger.info("email_verified", user_id=user.id) - - return True - - async def get_user_usage(self, user: User) -> Dict[str, Any]: - """Get current usage statistics for a user.""" - # Get current period - now = datetime.utcnow() - period_start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - - # Get or create usage record - usage = await self.db.get_user_usage(user.id, period_start) - - if not usage: - usage = await self._initialize_usage_record(user) - - # Get subscription limits - subscription = user.current_subscription - search_limit = subscription.search_limit if subscription else 1000 - scrape_limit = subscription.scrape_limit if subscription else 10000 - - return { - "period": { - "start": period_start.isoformat(), - "end": (period_start + timedelta(days=30)).isoformat() - }, - "searches": { - "used": usage.search_count, - "limit": search_limit, - "remaining": (search_limit - usage.search_count) if search_limit else None, - "unlimited": search_limit is None - }, - "scrapes": { - "used": usage.scrape_count, - "limit": scrape_limit, - "remaining": (scrape_limit - usage.scrape_count) if scrape_limit else None, - "unlimited": scrape_limit is None - }, - "api_calls": usage.api_calls, - "usage_by_engine": usage.usage_by_engine, - "usage_by_day": usage.usage_by_day - } - - async def check_usage_limits(self, user: User, search: bool = False, scrape: bool = False) -> bool: - """Check if user has exceeded usage limits.""" - # Get current usage - usage_data = await self.get_user_usage(user) - - if search: - if not usage_data["searches"]["unlimited"]: - if usage_data["searches"]["remaining"] <= 0: - return False - - if scrape: - if not usage_data["scrapes"]["unlimited"]: - if usage_data["scrapes"]["remaining"] <= 0: - return False - - return True - - async def increment_usage( - self, - user: User, - search_count: int = 0, - scrape_count: int = 0, - engine: Optional[str] = None - ): - """Increment usage counters for a user.""" - now = datetime.utcnow() - period_start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - - usage = await self.db.get_user_usage(user.id, period_start) - if not usage: - usage = await self._initialize_usage_record(user) - - # Update counts - usage.search_count += search_count - usage.scrape_count += scrape_count - usage.api_calls += 1 - - # Update usage by engine - if engine and search_count > 0: - if engine not in usage.usage_by_engine: - usage.usage_by_engine[engine] = 0 - usage.usage_by_engine[engine] += search_count - - # Update usage by day - today = now.date().isoformat() - if today not in usage.usage_by_day: - usage.usage_by_day[today] = 0 - usage.usage_by_day[today] += search_count + scrape_count - - # Check for overages - subscription = user.current_subscription - if subscription: - if subscription.search_limit and usage.search_count > subscription.search_limit: - usage.search_overage = usage.search_count - subscription.search_limit - if subscription.scrape_limit and usage.scrape_count > subscription.scrape_limit: - usage.scrape_overage = usage.scrape_count - subscription.scrape_limit - - await self.db.update_usage_record(usage) - - async def _initialize_usage_record(self, user: User) -> UsageRecord: - """Initialize usage record for current period.""" - now = datetime.utcnow() - period_start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - period_end = (period_start + timedelta(days=30)).replace(hour=23, minute=59, second=59) - - usage = UsageRecord( - user_id=user.id, - period_start=period_start, - period_end=period_end, - search_count=0, - scrape_count=0, - api_calls=0, - usage_by_engine={}, - usage_by_day={} - ) - - return await self.db.create_usage_record(usage) - - -# Singleton instance -_auth_service: Optional[AuthService] = None - - -async def get_auth_service(db_service: DatabaseService = Depends(get_database_service)) -> AuthService: - """Get or create auth service instance.""" - global _auth_service - - if _auth_service is None: - _auth_service = AuthService(db_service) - - return _auth_service diff --git a/apps/backend/app/services/batch_operations.py b/apps/backend/app/services/batch_operations.py deleted file mode 100644 index 769443b..0000000 --- a/apps/backend/app/services/batch_operations.py +++ /dev/null @@ -1,628 +0,0 @@ -""" -Advanced batch processing operations inspired by Firecrawl's batch capabilities. - -Provides sophisticated batch scraping, extraction, and processing with: -- Intelligent job scheduling and prioritization -- Progress tracking and status updates -- Error handling and retry logic -- Resource management and throttling -""" - -import asyncio -import time -import uuid -from typing import Dict, List, Optional, Any, Union, Callable, Awaitable -from enum import Enum -from dataclasses import dataclass, field -from datetime import datetime, timedelta -import structlog -from collections import deque - -from app.config import get_settings -from app.models.requests import ScrapingConfig, BatchSearchRequest -from app.models.responses import ScrapedContent, SearchResult -from app.services.enhanced_scraping import get_enhanced_scraping_service -from app.services.multi_search import get_multi_search_service -from app.services.dispatcher import create_dispatcher - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class BatchJobStatus(Enum): - """Status of batch jobs.""" - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - CANCELLED = "cancelled" - PAUSED = "paused" - - -class BatchJobType(Enum): - """Types of batch jobs.""" - SCRAPE = "scrape" - SEARCH = "search" - EXTRACT = "extract" - CRAWL = "crawl" - - -@dataclass -class BatchJobConfig: - """Configuration for batch operations.""" - job_type: BatchJobType - urls: List[str] = field(default_factory=list) - config: Optional[Dict[str, Any]] = None - priority: int = 10 # Lower = higher priority - max_retries: int = 3 - retry_delay: int = 5 # seconds - timeout_per_url: int = 30 # seconds - max_concurrent: int = 10 - webhook_url: Optional[str] = None - metadata: Dict[str, Any] = field(default_factory=dict) - - -@dataclass -class BatchJobProgress: - """Progress tracking for batch jobs.""" - total_urls: int = 0 - completed_urls: int = 0 - failed_urls: int = 0 - skipped_urls: int = 0 - current_url: Optional[str] = None - estimated_completion: Optional[datetime] = None - start_time: Optional[datetime] = None - end_time: Optional[datetime] = None - - @property - def completion_percentage(self) -> float: - """Calculate completion percentage.""" - if self.total_urls == 0: - return 0.0 - return (self.completed_urls / self.total_urls) * 100 - - @property - def is_complete(self) -> bool: - """Check if job is complete.""" - return (self.completed_urls + self.failed_urls + self.skipped_urls) >= self.total_urls - - -@dataclass -class BatchJobResult: - """Result of a batch job.""" - job_id: str - status: BatchJobStatus - results: List[Any] = field(default_factory=list) - errors: List[str] = field(default_factory=list) - progress: Optional[BatchJobProgress] = None - metadata: Dict[str, Any] = field(default_factory=dict) - created_at: Optional[datetime] = None - updated_at: Optional[datetime] = None - - -class BatchJob: - """Represents a single batch job with full lifecycle management.""" - - def __init__(self, job_id: str, config: BatchJobConfig): - self.job_id = job_id - self.config = config - self.status = BatchJobStatus.PENDING - self.progress = BatchJobProgress(total_urls=len(config.urls)) - self.results: List[Any] = [] - self.errors: List[str] = [] - self.created_at = datetime.utcnow() - self.updated_at = datetime.utcnow() - self.retry_counts: Dict[str, int] = {} - self.failed_urls: List[str] = [] - self.processing_times: deque = deque(maxlen=10) # Track recent processing times - - def update_status(self, status: BatchJobStatus): - """Update job status with timestamp.""" - self.status = status - self.updated_at = datetime.utcnow() - - if status == BatchJobStatus.PROCESSING and not self.progress.start_time: - self.progress.start_time = datetime.utcnow() - elif status in [BatchJobStatus.COMPLETED, BatchJobStatus.FAILED, BatchJobStatus.CANCELLED]: - self.progress.end_time = datetime.utcnow() - - def add_result(self, url: str, result: Any, processing_time: float): - """Add successful result.""" - self.results.append(result) - self.progress.completed_urls += 1 - self.processing_times.append(processing_time) - self._update_estimated_completion() - self.updated_at = datetime.utcnow() - - def add_error(self, url: str, error: str): - """Add error result.""" - self.errors.append(f"{url}: {error}") - self.progress.failed_urls += 1 - self.failed_urls.append(url) - self.updated_at = datetime.utcnow() - - def skip_url(self, url: str, reason: str): - """Skip URL with reason.""" - self.progress.skipped_urls += 1 - self.errors.append(f"{url}: Skipped - {reason}") - self.updated_at = datetime.utcnow() - - def _update_estimated_completion(self): - """Update estimated completion time based on current progress.""" - if len(self.processing_times) > 0 and self.progress.completed_urls > 0: - avg_time = sum(self.processing_times) / len(self.processing_times) - remaining_urls = self.progress.total_urls - self.progress.completed_urls - self.progress.failed_urls - self.progress.skipped_urls - - if remaining_urls > 0: - estimated_seconds = remaining_urls * avg_time - self.progress.estimated_completion = datetime.utcnow() + timedelta(seconds=estimated_seconds) - - def should_retry_url(self, url: str) -> bool: - """Check if URL should be retried.""" - retry_count = self.retry_counts.get(url, 0) - return retry_count < self.config.max_retries - - def increment_retry(self, url: str): - """Increment retry count for URL.""" - self.retry_counts[url] = self.retry_counts.get(url, 0) + 1 - - def to_result(self) -> BatchJobResult: - """Convert to result object.""" - return BatchJobResult( - job_id=self.job_id, - status=self.status, - results=self.results, - errors=self.errors, - progress=self.progress, - metadata=self.config.metadata, - created_at=self.created_at, - updated_at=self.updated_at - ) - - -class BatchOperationService: - """ - Advanced batch operation service with intelligent job management. - - Features: - - Concurrent processing with resource management - - Priority-based job scheduling - - Automatic retry logic with exponential backoff - - Progress tracking and status updates - - Webhook notifications - - Job pause/resume/cancel capabilities - """ - - def __init__(self): - """Initialize batch operation service.""" - self.active_jobs: Dict[str, BatchJob] = {} - self.job_queue: asyncio.PriorityQueue = asyncio.PriorityQueue() - self.completed_jobs: Dict[str, BatchJob] = {} - self.worker_tasks: List[asyncio.Task] = [] - self.max_concurrent_jobs = getattr(settings, 'batch_max_concurrent_jobs', 5) - self.max_workers = getattr(settings, 'batch_max_workers', 10) - self.stats = { - "total_jobs": 0, - "completed_jobs": 0, - "failed_jobs": 0, - "total_urls_processed": 0 - } - self.is_running = False - - # Initialize services - self.dispatcher = create_dispatcher( - dispatcher_type="memory_adaptive", - max_concurrent=settings.scraping_max_concurrent - ) - - async def start(self): - """Start the batch processing service.""" - if self.is_running: - return - - self.is_running = True - logger.info("batch_service_starting", max_concurrent_jobs=self.max_concurrent_jobs) - - # Start worker tasks - for i in range(self.max_workers): - worker_task = asyncio.create_task(self._worker(f"worker-{i}")) - self.worker_tasks.append(worker_task) - - async def stop(self): - """Stop the batch processing service.""" - self.is_running = False - logger.info("batch_service_stopping") - - # Cancel all worker tasks - for task in self.worker_tasks: - task.cancel() - - # Wait for tasks to complete - await asyncio.gather(*self.worker_tasks, return_exceptions=True) - self.worker_tasks.clear() - - # Cleanup dispatcher - if self.dispatcher: - await self.dispatcher.cleanup() - - async def submit_batch_scrape( - self, - urls: List[str], - config: Optional[ScrapingConfig] = None, - priority: int = 10, - webhook_url: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None - ) -> str: - """ - Submit batch scraping job. - - Args: - urls: List of URLs to scrape - config: Scraping configuration - priority: Job priority (lower = higher priority) - webhook_url: Optional webhook for status updates - metadata: Optional job metadata - - Returns: - Job ID for tracking - """ - job_id = str(uuid.uuid4()) - - batch_config = BatchJobConfig( - job_type=BatchJobType.SCRAPE, - urls=urls, - config=config.dict() if config else {}, - priority=priority, - webhook_url=webhook_url, - metadata=metadata or {} - ) - - job = BatchJob(job_id, batch_config) - self.active_jobs[job_id] = job - self.stats["total_jobs"] += 1 - - # Add to queue - await self.job_queue.put((priority, time.time(), job)) - - logger.info("batch_scrape_submitted", - job_id=job_id, - urls=len(urls), - priority=priority) - - return job_id - - async def submit_batch_search( - self, - queries: List[str], - search_config: Optional[Dict[str, Any]] = None, - priority: int = 10, - webhook_url: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None - ) -> str: - """Submit batch search job.""" - job_id = str(uuid.uuid4()) - - batch_config = BatchJobConfig( - job_type=BatchJobType.SEARCH, - urls=queries, # Using urls field for queries - config=search_config or {}, - priority=priority, - webhook_url=webhook_url, - metadata=metadata or {} - ) - - job = BatchJob(job_id, batch_config) - self.active_jobs[job_id] = job - self.stats["total_jobs"] += 1 - - await self.job_queue.put((priority, time.time(), job)) - - logger.info("batch_search_submitted", - job_id=job_id, - queries=len(queries), - priority=priority) - - return job_id - - async def get_job_status(self, job_id: str) -> Optional[BatchJobResult]: - """Get status of specific job.""" - if job_id in self.active_jobs: - return self.active_jobs[job_id].to_result() - elif job_id in self.completed_jobs: - return self.completed_jobs[job_id].to_result() - else: - return None - - async def cancel_job(self, job_id: str) -> bool: - """Cancel a job.""" - if job_id in self.active_jobs: - job = self.active_jobs[job_id] - job.update_status(BatchJobStatus.CANCELLED) - logger.info("job_cancelled", job_id=job_id) - return True - return False - - async def pause_job(self, job_id: str) -> bool: - """Pause a job.""" - if job_id in self.active_jobs: - job = self.active_jobs[job_id] - if job.status == BatchJobStatus.PROCESSING: - job.update_status(BatchJobStatus.PAUSED) - logger.info("job_paused", job_id=job_id) - return True - return False - - async def resume_job(self, job_id: str) -> bool: - """Resume a paused job.""" - if job_id in self.active_jobs: - job = self.active_jobs[job_id] - if job.status == BatchJobStatus.PAUSED: - job.update_status(BatchJobStatus.PROCESSING) - logger.info("job_resumed", job_id=job_id) - return True - return False - - async def _worker(self, worker_id: str): - """Worker task for processing jobs.""" - logger.info("worker_started", worker_id=worker_id) - - while self.is_running: - try: - # Get job from queue with timeout - try: - priority, timestamp, job = await asyncio.wait_for( - self.job_queue.get(), timeout=1.0 - ) - except asyncio.TimeoutError: - continue - - # Skip if too many concurrent jobs - active_processing = sum( - 1 for j in self.active_jobs.values() - if j.status == BatchJobStatus.PROCESSING - ) - - if active_processing >= self.max_concurrent_jobs: - # Put job back in queue - await self.job_queue.put((priority, timestamp, job)) - await asyncio.sleep(1) - continue - - # Skip cancelled jobs - if job.status == BatchJobStatus.CANCELLED: - continue - - # Process the job - logger.info("worker_processing_job", worker_id=worker_id, job_id=job.job_id) - await self._process_job(job) - - # Move to completed jobs - self.completed_jobs[job.job_id] = job - if job.job_id in self.active_jobs: - del self.active_jobs[job.job_id] - - # Update stats - if job.status == BatchJobStatus.COMPLETED: - self.stats["completed_jobs"] += 1 - else: - self.stats["failed_jobs"] += 1 - - self.stats["total_urls_processed"] += len(job.config.urls) - - except Exception as e: - logger.error("worker_error", worker_id=worker_id, error=str(e)) - await asyncio.sleep(1) - - logger.info("worker_stopped", worker_id=worker_id) - - async def _process_job(self, job: BatchJob): - """Process a single batch job.""" - job.update_status(BatchJobStatus.PROCESSING) - - try: - if job.config.job_type == BatchJobType.SCRAPE: - await self._process_scrape_job(job) - elif job.config.job_type == BatchJobType.SEARCH: - await self._process_search_job(job) - elif job.config.job_type == BatchJobType.EXTRACT: - await self._process_extract_job(job) - elif job.config.job_type == BatchJobType.CRAWL: - await self._process_crawl_job(job) - else: - raise ValueError(f"Unknown job type: {job.config.job_type}") - - job.update_status(BatchJobStatus.COMPLETED) - - except Exception as e: - logger.error("job_processing_failed", job_id=job.job_id, error=str(e)) - job.add_error("general", str(e)) - job.update_status(BatchJobStatus.FAILED) - - # Send webhook notification if configured - if job.config.webhook_url: - await self._send_webhook(job) - - async def _process_scrape_job(self, job: BatchJob): - """Process batch scraping job.""" - scraping_service = await get_enhanced_scraping_service() - - # Create scraping config - config = ScrapingConfig(**job.config.config) if job.config.config else ScrapingConfig(urls=[]) - - # Process URLs in batches to manage resources - batch_size = min(job.config.max_concurrent, 10) - - for i in range(0, len(job.config.urls), batch_size): - if job.status == BatchJobStatus.CANCELLED: - break - - # Wait if job is paused - while job.status == BatchJobStatus.PAUSED: - await asyncio.sleep(1) - - batch_urls = job.config.urls[i:i + batch_size] - - # Process batch - tasks = [] - for url in batch_urls: - task = self._scrape_single_url(job, url, config, scraping_service) - tasks.append(task) - - # Execute batch - await asyncio.gather(*tasks, return_exceptions=True) - - async def _scrape_single_url(self, job: BatchJob, url: str, config: ScrapingConfig, service): - """Scrape a single URL with retry logic.""" - start_time = time.time() - - for attempt in range(job.config.max_retries + 1): - try: - # Update current URL in progress - job.progress.current_url = url - - # Scrape the URL - config.urls = [url] # Set current URL - results = await service.scrape_urls_enhanced([url], config) - - if results and len(results) > 0: - result = results[0] - if result.extraction_success: - processing_time = time.time() - start_time - job.add_result(url, result.dict(), processing_time) - return - - # If we reach here, scraping failed - if attempt == job.config.max_retries: - job.add_error(url, "Scraping failed after all retries") - else: - await asyncio.sleep(job.config.retry_delay * (attempt + 1)) - - except Exception as e: - if attempt == job.config.max_retries: - job.add_error(url, str(e)) - else: - logger.warning("scrape_attempt_failed", - url=url, - attempt=attempt + 1, - error=str(e)) - await asyncio.sleep(job.config.retry_delay * (attempt + 1)) - - async def _process_search_job(self, job: BatchJob): - """Process batch search job.""" - search_service = await get_multi_search_service() - - for query in job.config.urls: # Using urls field for queries - if job.status == BatchJobStatus.CANCELLED: - break - - while job.status == BatchJobStatus.PAUSED: - await asyncio.sleep(1) - - try: - start_time = time.time() - job.progress.current_url = query - - # Perform search - from app.services.multi_search import SearchOptions - options = SearchOptions( - query=query, - num_results=job.config.config.get("max_results", 10), - lang=job.config.config.get("language", "en"), - country=job.config.config.get("country", "us") - ) - - results = await search_service.search(options) - processing_time = time.time() - start_time - - job.add_result(query, [r.dict() for r in results], processing_time) - - except Exception as e: - job.add_error(query, str(e)) - - async def _process_extract_job(self, job: BatchJob): - """Process batch extraction job.""" - # Implementation would handle batch extraction - for url in job.config.urls: - if job.status == BatchJobStatus.CANCELLED: - break - job.add_result(url, {"extracted": True}, 1.0) - - async def _process_crawl_job(self, job: BatchJob): - """Process batch crawling job.""" - # Implementation would handle batch crawling - for url in job.config.urls: - if job.status == BatchJobStatus.CANCELLED: - break - job.add_result(url, {"crawled": True}, 1.0) - - async def _send_webhook(self, job: BatchJob): - """Send webhook notification for job completion.""" - if not job.config.webhook_url: - return - - try: - import httpx - async with httpx.AsyncClient() as client: - payload = { - "job_id": job.job_id, - "status": job.status.value, - "progress": { - "total_urls": job.progress.total_urls, - "completed_urls": job.progress.completed_urls, - "failed_urls": job.progress.failed_urls, - "completion_percentage": job.progress.completion_percentage - }, - "results_count": len(job.results), - "errors_count": len(job.errors), - "completed_at": job.updated_at.isoformat() if job.updated_at else None - } - - response = await client.post( - job.config.webhook_url, - json=payload, - timeout=10 - ) - - if response.status_code == 200: - logger.info("webhook_sent", job_id=job.job_id, url=job.config.webhook_url) - else: - logger.warning("webhook_failed", - job_id=job.job_id, - status_code=response.status_code) - - except Exception as e: - logger.error("webhook_error", job_id=job.job_id, error=str(e)) - - async def get_service_stats(self) -> Dict[str, Any]: - """Get comprehensive service statistics.""" - active_jobs_stats = {} - for job_id, job in self.active_jobs.items(): - active_jobs_stats[job_id] = { - "status": job.status.value, - "progress": job.progress.completion_percentage, - "created_at": job.created_at.isoformat() - } - - return { - "stats": self.stats, - "active_jobs": len(self.active_jobs), - "completed_jobs": len(self.completed_jobs), - "queue_size": self.job_queue.qsize(), - "workers_running": len(self.worker_tasks), - "active_jobs_details": active_jobs_stats - } - - -# Singleton instance -_batch_service: Optional[BatchOperationService] = None - - -async def get_batch_service() -> BatchOperationService: - """Get or create batch operation service instance.""" - global _batch_service - - if _batch_service is None: - _batch_service = BatchOperationService() - await _batch_service.start() - - return _batch_service diff --git a/apps/backend/app/services/browser_config.py b/apps/backend/app/services/browser_config.py deleted file mode 100644 index 533d8b7..0000000 --- a/apps/backend/app/services/browser_config.py +++ /dev/null @@ -1,578 +0,0 @@ -""" -Advanced browser configuration and management inspired by crawl4ai. - -This module provides comprehensive browser configuration options: -- BrowserConfig: Main browser configuration class -- GeolocationConfig: Geolocation settings -- ProxyConfig: Proxy configuration -- UserAgentConfig: User agent management -""" - -import os -import json -import random -from typing import Dict, List, Optional, Any, Union -from dataclasses import dataclass, field -from enum import Enum - -import structlog - -logger = structlog.get_logger(__name__) - - -class BrowserType(str, Enum): - """Supported browser types.""" - CHROMIUM = "chromium" - FIREFOX = "firefox" - WEBKIT = "webkit" - - -class DeviceType(str, Enum): - """Device types for emulation.""" - DESKTOP = "desktop" - MOBILE = "mobile" - TABLET = "tablet" - - -@dataclass -class GeolocationConfig: - """Configuration for browser geolocation settings.""" - - latitude: float - longitude: float - accuracy: float = 0.0 - - @classmethod - def from_dict(cls, geo_dict: Dict[str, Any]) -> "GeolocationConfig": - """Create GeolocationConfig from dictionary.""" - return cls( - latitude=geo_dict.get("latitude"), - longitude=geo_dict.get("longitude"), - accuracy=geo_dict.get("accuracy", 0.0) - ) - - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary representation.""" - return { - "latitude": self.latitude, - "longitude": self.longitude, - "accuracy": self.accuracy - } - - def clone(self, **kwargs) -> "GeolocationConfig": - """Create a copy with updated values.""" - config_dict = self.to_dict() - config_dict.update(kwargs) - return GeolocationConfig.from_dict(config_dict) - - -@dataclass -class ProxyConfig: - """Configuration for a single proxy.""" - - server: str - username: Optional[str] = None - password: Optional[str] = None - ip: Optional[str] = None - - def __post_init__(self): - """Extract IP from server if not provided.""" - if not self.ip: - self.ip = self._extract_ip_from_server() - - def _extract_ip_from_server(self) -> Optional[str]: - """Extract IP address from server URL.""" - try: - if "://" in self.server: - parts = self.server.split("://")[1].split(":") - return parts[0] - else: - parts = self.server.split(":") - return parts[0] - except Exception: - return None - - @classmethod - def from_string(cls, proxy_str: str) -> "ProxyConfig": - """Create ProxyConfig from string format 'ip:port:username:password'.""" - parts = proxy_str.split(":") - if len(parts) == 4: # ip:port:username:password - ip, port, username, password = parts - return cls( - server=f"http://{ip}:{port}", - username=username, - password=password, - ip=ip - ) - elif len(parts) == 2: # ip:port only - ip, port = parts - return cls( - server=f"http://{ip}:{port}", - ip=ip - ) - else: - raise ValueError(f"Invalid proxy string format: {proxy_str}") - - @classmethod - def from_dict(cls, proxy_dict: Dict[str, Any]) -> "ProxyConfig": - """Create ProxyConfig from dictionary.""" - return cls( - server=proxy_dict.get("server"), - username=proxy_dict.get("username"), - password=proxy_dict.get("password"), - ip=proxy_dict.get("ip") - ) - - @classmethod - def from_env(cls, env_var: str = "PROXIES") -> List["ProxyConfig"]: - """Load proxies from environment variable.""" - proxies = [] - try: - proxy_list = os.getenv(env_var, "").split(",") - for proxy in proxy_list: - proxy = proxy.strip() - if proxy: - proxies.append(cls.from_string(proxy)) - except Exception as e: - logger.warning(f"Error loading proxies from env {env_var}: {str(e)}") - - return proxies - - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary representation.""" - return { - "server": self.server, - "username": self.username, - "password": self.password, - "ip": self.ip - } - - -@dataclass -class UserAgentConfig: - """Configuration for user agent management.""" - - user_agent: Optional[str] = None - platform: Optional[str] = None - device_type: DeviceType = DeviceType.DESKTOP - randomize: bool = False - - # Common user agents for different platforms - DESKTOP_USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0" - ] - - MOBILE_USER_AGENTS = [ - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1", - "Mozilla/5.0 (Linux; Android 14; SM-G998B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36", - "Mozilla/5.0 (Linux; Android 14; Pixel 8 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36" - ] - - TABLET_USER_AGENTS = [ - "Mozilla/5.0 (iPad; CPU OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1", - "Mozilla/5.0 (Linux; Android 14; SM-T970) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - ] - - def get_user_agent(self) -> str: - """Get user agent string based on configuration.""" - if self.user_agent: - return self.user_agent - - if self.randomize: - agents_list = self._get_agents_for_device_type() - return random.choice(agents_list) - else: - # Return default for device type - agents_list = self._get_agents_for_device_type() - return agents_list[0] if agents_list else self.DESKTOP_USER_AGENTS[0] - - def _get_agents_for_device_type(self) -> List[str]: - """Get user agent list for device type.""" - if self.device_type == DeviceType.MOBILE: - return self.MOBILE_USER_AGENTS - elif self.device_type == DeviceType.TABLET: - return self.TABLET_USER_AGENTS - else: - return self.DESKTOP_USER_AGENTS - - -@dataclass -class BrowserConfig: - """ - Comprehensive browser configuration for advanced web scraping. - - This class centralizes all browser-related parameters and settings - for consistent configuration across different scraping scenarios. - """ - - # Basic browser settings - browser_type: BrowserType = BrowserType.CHROMIUM - headless: bool = True - verbose: bool = False - - # Window and viewport settings - viewport_width: int = 1920 - viewport_height: int = 1080 - window_width: Optional[int] = None - window_height: Optional[int] = None - device_scale_factor: float = 1.0 - - # User agent and device emulation - user_agent_config: Optional[UserAgentConfig] = None - device_type: DeviceType = DeviceType.DESKTOP - - # Proxy and network settings - proxy_config: Optional[ProxyConfig] = None - ignore_https_errors: bool = True - bypass_csp: bool = True - - # Geolocation settings - geolocation_config: Optional[GeolocationConfig] = None - - # Browser profile and persistence - user_data_dir: Optional[str] = None - use_persistent_context: bool = False - profile_name: Optional[str] = None - - # Performance and resource settings - javascript_enabled: bool = True - images_enabled: bool = True - css_enabled: bool = True - plugins_enabled: bool = False - webgl_enabled: bool = True - - # Security and privacy settings - accept_downloads: bool = False - permissions: List[str] = field(default_factory=list) - locale: str = "en-US" - timezone: Optional[str] = None - - # Browser launch arguments - browser_args: List[str] = field(default_factory=list) - chromium_sandbox: bool = True - - # Timeouts - page_timeout: int = 30000 # milliseconds - navigation_timeout: int = 30000 # milliseconds - - # Extra HTTP headers - extra_headers: Dict[str, str] = field(default_factory=dict) - - # Cookie and session settings - accept_cookies: bool = True - cookie_file: Optional[str] = None - - # Screenshot and media settings - full_page_screenshot: bool = False - screenshot_quality: int = 80 - - def __post_init__(self): - """Post-initialization processing.""" - # Set default user agent config if not provided - if self.user_agent_config is None: - self.user_agent_config = UserAgentConfig(device_type=self.device_type) - - # Set default browser arguments for security and performance - if not self.browser_args: - self.browser_args = self._get_default_browser_args() - - # Set window size if not specified - if self.window_width is None: - self.window_width = self.viewport_width - if self.window_height is None: - self.window_height = self.viewport_height - - def _get_default_browser_args(self) -> List[str]: - """Get default browser arguments for security and performance.""" - args = [ - "--no-first-run", - "--no-default-browser-check", - "--disable-extensions", - "--disable-plugins", - "--disable-default-apps", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-features=TranslateUI", - ] - - if not self.chromium_sandbox: - args.extend([ - "--no-sandbox", - "--disable-setuid-sandbox" - ]) - - if self.device_type == DeviceType.MOBILE: - args.extend([ - "--enable-touch-events", - "--enable-viewport" - ]) - - return args - - def get_launch_options(self) -> Dict[str, Any]: - """Get browser launch options dictionary.""" - options = { - "headless": self.headless, - "args": self.browser_args, - "ignore_https_errors": self.ignore_https_errors, - "timeout": self.page_timeout - } - - if self.user_data_dir: - options["user_data_dir"] = self.user_data_dir - - if self.proxy_config: - options["proxy"] = { - "server": self.proxy_config.server - } - if self.proxy_config.username: - options["proxy"]["username"] = self.proxy_config.username - if self.proxy_config.password: - options["proxy"]["password"] = self.proxy_config.password - - return options - - def get_context_options(self) -> Dict[str, Any]: - """Get browser context options dictionary.""" - options = { - "viewport": { - "width": self.viewport_width, - "height": self.viewport_height - }, - "user_agent": self.user_agent_config.get_user_agent() if self.user_agent_config else None, - "locale": self.locale, - "timezone_id": self.timezone, - "permissions": self.permissions, - "extra_http_headers": self.extra_headers, - "bypass_csp": self.bypass_csp, - "javascript_enabled": self.javascript_enabled, - "accept_downloads": self.accept_downloads - } - - if self.geolocation_config: - options["geolocation"] = self.geolocation_config.to_dict() - - # Remove None values - return {k: v for k, v in options.items() if v is not None} - - def get_page_options(self) -> Dict[str, Any]: - """Get page-specific options.""" - return { - "timeout": self.navigation_timeout, - "wait_until": "domcontentloaded" - } - - def clone(self, **kwargs) -> "BrowserConfig": - """Create a copy of this configuration with updated values.""" - # Create a copy of current config - current_dict = self.to_dict() - current_dict.update(kwargs) - return BrowserConfig.from_dict(current_dict) - - def to_dict(self) -> Dict[str, Any]: - """Convert configuration to dictionary.""" - return { - "browser_type": self.browser_type.value, - "headless": self.headless, - "verbose": self.verbose, - "viewport_width": self.viewport_width, - "viewport_height": self.viewport_height, - "window_width": self.window_width, - "window_height": self.window_height, - "device_scale_factor": self.device_scale_factor, - "user_agent_config": { - "user_agent": self.user_agent_config.user_agent, - "platform": self.user_agent_config.platform, - "device_type": self.user_agent_config.device_type.value, - "randomize": self.user_agent_config.randomize - } if self.user_agent_config else None, - "device_type": self.device_type.value, - "proxy_config": self.proxy_config.to_dict() if self.proxy_config else None, - "ignore_https_errors": self.ignore_https_errors, - "bypass_csp": self.bypass_csp, - "geolocation_config": self.geolocation_config.to_dict() if self.geolocation_config else None, - "user_data_dir": self.user_data_dir, - "use_persistent_context": self.use_persistent_context, - "profile_name": self.profile_name, - "javascript_enabled": self.javascript_enabled, - "images_enabled": self.images_enabled, - "css_enabled": self.css_enabled, - "plugins_enabled": self.plugins_enabled, - "webgl_enabled": self.webgl_enabled, - "accept_downloads": self.accept_downloads, - "permissions": self.permissions, - "locale": self.locale, - "timezone": self.timezone, - "browser_args": self.browser_args, - "chromium_sandbox": self.chromium_sandbox, - "page_timeout": self.page_timeout, - "navigation_timeout": self.navigation_timeout, - "extra_headers": self.extra_headers, - "accept_cookies": self.accept_cookies, - "cookie_file": self.cookie_file, - "full_page_screenshot": self.full_page_screenshot, - "screenshot_quality": self.screenshot_quality - } - - @classmethod - def from_dict(cls, config_dict: Dict[str, Any]) -> "BrowserConfig": - """Create BrowserConfig from dictionary.""" - # Handle nested configs - user_agent_config = None - if config_dict.get("user_agent_config"): - ua_dict = config_dict["user_agent_config"] - user_agent_config = UserAgentConfig( - user_agent=ua_dict.get("user_agent"), - platform=ua_dict.get("platform"), - device_type=DeviceType(ua_dict.get("device_type", "desktop")), - randomize=ua_dict.get("randomize", False) - ) - - proxy_config = None - if config_dict.get("proxy_config"): - proxy_config = ProxyConfig.from_dict(config_dict["proxy_config"]) - - geolocation_config = None - if config_dict.get("geolocation_config"): - geolocation_config = GeolocationConfig.from_dict(config_dict["geolocation_config"]) - - # Create config with processed nested objects - processed_dict = config_dict.copy() - processed_dict["browser_type"] = BrowserType(config_dict.get("browser_type", "chromium")) - processed_dict["device_type"] = DeviceType(config_dict.get("device_type", "desktop")) - processed_dict["user_agent_config"] = user_agent_config - processed_dict["proxy_config"] = proxy_config - processed_dict["geolocation_config"] = geolocation_config - - return cls(**{k: v for k, v in processed_dict.items() if v is not None}) - - -# Predefined browser configurations for common use cases -def get_stealth_browser_config() -> BrowserConfig: - """Get browser configuration optimized for stealth scraping.""" - return BrowserConfig( - headless=True, - user_agent_config=UserAgentConfig(randomize=True), - ignore_https_errors=True, - bypass_csp=True, - chromium_sandbox=False, - browser_args=[ - "--no-first-run", - "--no-default-browser-check", - "--disable-extensions", - "--disable-plugins", - "--disable-default-apps", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-features=TranslateUI,BlinkGenPropertyTrees", - "--disable-ipc-flooding-protection", - "--disable-blink-features=AutomationControlled", - "--no-sandbox", - "--disable-setuid-sandbox", - "--disable-dev-shm-usage" - ], - extra_headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "DNT": "1", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1" - } - ) - - -def get_mobile_browser_config() -> BrowserConfig: - """Get browser configuration for mobile device emulation.""" - return BrowserConfig( - device_type=DeviceType.MOBILE, - viewport_width=414, - viewport_height=896, - device_scale_factor=3.0, - user_agent_config=UserAgentConfig( - device_type=DeviceType.MOBILE, - randomize=False - ), - browser_args=[ - "--enable-touch-events", - "--enable-viewport", - "--disable-extensions", - "--no-first-run" - ] - ) - - -def get_high_performance_browser_config() -> BrowserConfig: - """Get browser configuration optimized for performance.""" - return BrowserConfig( - headless=True, - images_enabled=False, # Skip image loading - css_enabled=False, # Skip CSS loading - plugins_enabled=False, - webgl_enabled=False, - javascript_enabled=True, # Keep JS for dynamic content - browser_args=[ - "--disable-extensions", - "--disable-plugins", - "--disable-default-apps", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-features=TranslateUI", - "--disable-sync", - "--disable-background-networking", - "--disable-background-mode", - "--disable-client-side-phishing-detection", - "--disable-component-update", - "--disable-default-apps", - "--no-first-run", - "--no-default-browser-check" - ] - ) - - -# Utility functions -def create_browser_config_from_env() -> BrowserConfig: - """Create browser configuration from environment variables.""" - config = BrowserConfig() - - # Basic settings - if os.getenv("BROWSER_HEADLESS"): - config.headless = os.getenv("BROWSER_HEADLESS").lower() == "true" - - if os.getenv("BROWSER_TYPE"): - config.browser_type = BrowserType(os.getenv("BROWSER_TYPE")) - - # Viewport settings - if os.getenv("VIEWPORT_WIDTH"): - config.viewport_width = int(os.getenv("VIEWPORT_WIDTH")) - - if os.getenv("VIEWPORT_HEIGHT"): - config.viewport_height = int(os.getenv("VIEWPORT_HEIGHT")) - - # User agent - if os.getenv("USER_AGENT"): - config.user_agent_config = UserAgentConfig(user_agent=os.getenv("USER_AGENT")) - - # Proxy settings - if os.getenv("PROXY_SERVER"): - config.proxy_config = ProxyConfig( - server=os.getenv("PROXY_SERVER"), - username=os.getenv("PROXY_USERNAME"), - password=os.getenv("PROXY_PASSWORD") - ) - - # Geolocation - if os.getenv("GEO_LATITUDE") and os.getenv("GEO_LONGITUDE"): - config.geolocation_config = GeolocationConfig( - latitude=float(os.getenv("GEO_LATITUDE")), - longitude=float(os.getenv("GEO_LONGITUDE")) - ) - - return config diff --git a/apps/backend/app/services/browser_profiler.py b/apps/backend/app/services/browser_profiler.py deleted file mode 100644 index 33c5df9..0000000 --- a/apps/backend/app/services/browser_profiler.py +++ /dev/null @@ -1,513 +0,0 @@ -""" -Browser profiler for creating and managing browser profiles for identity-based crawling. - -This module provides comprehensive browser profile management: -- Interactive profile creation -- Profile persistence and reuse -- Identity-based crawling support -- Cross-platform browser profile management -- Profile validation and cleanup -""" - -import os -import json -import uuid -import time -import signal -import asyncio -import shutil -import subprocess -from typing import Dict, List, Optional, Any -from pathlib import Path -from dataclasses import dataclass, asdict - -import structlog - -from app.services.browser_config import BrowserConfig, BrowserType, DeviceType - -logger = structlog.get_logger(__name__) - - -@dataclass -class BrowserProfile: - """Browser profile configuration.""" - profile_id: str - name: str - browser_type: str - profile_path: str - created_at: float - last_used: Optional[float] = None - description: Optional[str] = None - tags: List[str] = None - user_data: Dict[str, Any] = None - - def __post_init__(self): - if self.tags is None: - self.tags = [] - if self.user_data is None: - self.user_data = {} - - -class BrowserProfiler: - """ - Browser profile manager for Crawl4AI-style identity-based crawling. - - Provides functionality to: - - Create browser profiles interactively - - List and manage existing profiles - - Generate BrowserConfig objects from profiles - - Clean up unused profiles - """ - - def __init__(self, profiles_dir: Optional[Path] = None): - """ - Initialize browser profiler. - - Args: - profiles_dir: Directory to store profiles (default: ~/.unsearch/profiles) - """ - if profiles_dir: - self.profiles_dir = Path(profiles_dir) - else: - home_dir = Path.home() / '.unsearch' - self.profiles_dir = home_dir / 'profiles' - - # Ensure profiles directory exists - self.profiles_dir.mkdir(parents=True, exist_ok=True) - - # Profile registry file - self.registry_file = self.profiles_dir / 'profiles.json' - - # Load existing profiles - self.profiles: Dict[str, BrowserProfile] = self._load_profiles() - - def _load_profiles(self) -> Dict[str, BrowserProfile]: - """Load profiles from registry file.""" - if not self.registry_file.exists(): - return {} - - try: - with open(self.registry_file, 'r') as f: - profiles_data = json.load(f) - - profiles = {} - for profile_id, profile_data in profiles_data.items(): - try: - profile = BrowserProfile(**profile_data) - profiles[profile_id] = profile - except Exception as e: - logger.warning(f"Failed to load profile {profile_id}: {str(e)}") - - return profiles - - except Exception as e: - logger.error(f"Failed to load profiles registry: {str(e)}") - return {} - - def _save_profiles(self): - """Save profiles to registry file.""" - try: - profiles_data = {} - for profile_id, profile in self.profiles.items(): - profiles_data[profile_id] = asdict(profile) - - with open(self.registry_file, 'w') as f: - json.dump(profiles_data, f, indent=2) - - except Exception as e: - logger.error(f"Failed to save profiles registry: {str(e)}") - - def create_profile(self, - name: str, - browser_type: str = "chromium", - description: str = None, - tags: List[str] = None, - interactive: bool = False) -> str: - """ - Create a new browser profile. - - Args: - name: Human-readable name for the profile - browser_type: Browser type (chromium, firefox, webkit) - description: Optional description - tags: Optional tags for organization - interactive: Whether to open browser for interactive setup - - Returns: - Profile ID - """ - profile_id = str(uuid.uuid4()) - - # Create profile directory - profile_path = self.profiles_dir / profile_id - profile_path.mkdir(exist_ok=True) - - # Create profile object - profile = BrowserProfile( - profile_id=profile_id, - name=name, - browser_type=browser_type, - profile_path=str(profile_path), - created_at=time.time(), - description=description, - tags=tags or [], - user_data={} - ) - - # Add to registry - self.profiles[profile_id] = profile - self._save_profiles() - - logger.info(f"Created browser profile: {name} ({profile_id})") - - # Interactive setup if requested - if interactive: - self._setup_profile_interactively(profile) - - return profile_id - - def _setup_profile_interactively(self, profile: BrowserProfile): - """Setup profile interactively by opening browser.""" - logger.info(f"Setting up profile interactively: {profile.name}") - - # This is a simplified version - in production you'd integrate with - # actual browser automation libraries like Playwright - - try: - # Create basic browser config for the profile - browser_config = self.get_browser_config(profile.profile_id) - - # In a real implementation, this would: - # 1. Launch browser with the profile - # 2. Allow user to log in to sites, set preferences - # 3. Wait for user to finish setup - # 4. Save the profile state - - logger.info( - f"Interactive setup for profile {profile.name} would open browser here. " - f"Profile data will be saved to: {profile.profile_path}" - ) - - # Mark as used - profile.last_used = time.time() - self._save_profiles() - - except Exception as e: - logger.error(f"Error in interactive setup: {str(e)}") - - def get_profile(self, profile_id: str) -> Optional[BrowserProfile]: - """Get profile by ID.""" - return self.profiles.get(profile_id) - - def get_profile_by_name(self, name: str) -> Optional[BrowserProfile]: - """Get profile by name.""" - for profile in self.profiles.values(): - if profile.name == name: - return profile - return None - - def list_profiles(self, tags: List[str] = None) -> List[BrowserProfile]: - """ - List all profiles, optionally filtered by tags. - - Args: - tags: Optional list of tags to filter by - - Returns: - List of matching profiles - """ - profiles = list(self.profiles.values()) - - if tags: - filtered_profiles = [] - for profile in profiles: - if any(tag in profile.tags for tag in tags): - filtered_profiles.append(profile) - profiles = filtered_profiles - - # Sort by last used, then by created - profiles.sort(key=lambda p: p.last_used or p.created_at, reverse=True) - return profiles - - def delete_profile(self, profile_id: str) -> bool: - """ - Delete a profile and its data. - - Args: - profile_id: Profile ID to delete - - Returns: - True if deleted successfully - """ - profile = self.profiles.get(profile_id) - if not profile: - logger.warning(f"Profile not found: {profile_id}") - return False - - try: - # Remove profile directory - profile_path = Path(profile.profile_path) - if profile_path.exists(): - shutil.rmtree(profile_path) - - # Remove from registry - del self.profiles[profile_id] - self._save_profiles() - - logger.info(f"Deleted profile: {profile.name} ({profile_id})") - return True - - except Exception as e: - logger.error(f"Error deleting profile {profile_id}: {str(e)}") - return False - - def get_browser_config(self, profile_id: str) -> Optional[BrowserConfig]: - """ - Create BrowserConfig for a profile. - - Args: - profile_id: Profile ID - - Returns: - BrowserConfig object or None if profile not found - """ - profile = self.get_profile(profile_id) - if not profile: - return None - - # Update last used time - profile.last_used = time.time() - self._save_profiles() - - # Create browser config - browser_type = BrowserType(profile.browser_type) - - config = BrowserConfig( - browser_type=browser_type, - user_data_dir=profile.profile_path, - use_persistent_context=True, - profile_name=profile.name, - headless=False, # Profiles typically used for interactive browsing - verbose=True - ) - - return config - - def validate_profiles(self) -> Dict[str, bool]: - """ - Validate all profiles and return status. - - Returns: - Dict mapping profile_id to validation status - """ - results = {} - - for profile_id, profile in self.profiles.items(): - try: - profile_path = Path(profile.profile_path) - is_valid = profile_path.exists() and profile_path.is_dir() - results[profile_id] = is_valid - - if not is_valid: - logger.warning(f"Profile {profile.name} has invalid path: {profile.profile_path}") - - except Exception as e: - logger.error(f"Error validating profile {profile_id}: {str(e)}") - results[profile_id] = False - - return results - - def cleanup_invalid_profiles(self) -> List[str]: - """ - Remove profiles with invalid paths. - - Returns: - List of removed profile IDs - """ - validation_results = self.validate_profiles() - removed_profiles = [] - - for profile_id, is_valid in validation_results.items(): - if not is_valid: - profile = self.profiles.get(profile_id) - if profile: - logger.info(f"Removing invalid profile: {profile.name} ({profile_id})") - del self.profiles[profile_id] - removed_profiles.append(profile_id) - - if removed_profiles: - self._save_profiles() - logger.info(f"Cleaned up {len(removed_profiles)} invalid profiles") - - return removed_profiles - - def export_profile(self, profile_id: str, export_path: Path) -> bool: - """ - Export a profile to a specified path. - - Args: - profile_id: Profile ID to export - export_path: Path to export to - - Returns: - True if exported successfully - """ - profile = self.get_profile(profile_id) - if not profile: - logger.error(f"Profile not found: {profile_id}") - return False - - try: - export_path.mkdir(parents=True, exist_ok=True) - - # Copy profile data - profile_path = Path(profile.profile_path) - if profile_path.exists(): - shutil.copytree(profile_path, export_path / "profile_data", dirs_exist_ok=True) - - # Export profile metadata - metadata_file = export_path / "profile_metadata.json" - with open(metadata_file, 'w') as f: - json.dump(asdict(profile), f, indent=2) - - logger.info(f"Exported profile {profile.name} to {export_path}") - return True - - except Exception as e: - logger.error(f"Error exporting profile {profile_id}: {str(e)}") - return False - - def import_profile(self, import_path: Path, new_name: str = None) -> Optional[str]: - """ - Import a profile from an exported path. - - Args: - import_path: Path to import from - new_name: Optional new name for the profile - - Returns: - New profile ID if imported successfully - """ - metadata_file = import_path / "profile_metadata.json" - if not metadata_file.exists(): - logger.error(f"Profile metadata not found at {import_path}") - return None - - try: - # Load metadata - with open(metadata_file, 'r') as f: - profile_data = json.load(f) - - # Generate new profile ID and path - new_profile_id = str(uuid.uuid4()) - new_profile_path = self.profiles_dir / new_profile_id - new_profile_path.mkdir(exist_ok=True) - - # Copy profile data - source_data = import_path / "profile_data" - if source_data.exists(): - shutil.copytree(source_data, new_profile_path, dirs_exist_ok=True) - - # Create new profile - profile = BrowserProfile( - profile_id=new_profile_id, - name=new_name or f"{profile_data.get('name', 'Imported')} (Imported)", - browser_type=profile_data.get('browser_type', 'chromium'), - profile_path=str(new_profile_path), - created_at=time.time(), - description=profile_data.get('description'), - tags=profile_data.get('tags', []), - user_data=profile_data.get('user_data', {}) - ) - - # Add to registry - self.profiles[new_profile_id] = profile - self._save_profiles() - - logger.info(f"Imported profile: {profile.name} ({new_profile_id})") - return new_profile_id - - except Exception as e: - logger.error(f"Error importing profile from {import_path}: {str(e)}") - # Cleanup on error - if 'new_profile_path' in locals() and new_profile_path.exists(): - shutil.rmtree(new_profile_path, ignore_errors=True) - return None - - def get_profile_stats(self) -> Dict[str, Any]: - """Get statistics about all profiles.""" - stats = { - 'total_profiles': len(self.profiles), - 'browser_types': {}, - 'profiles_by_age': { - 'last_week': 0, - 'last_month': 0, - 'older': 0 - }, - 'most_used_tags': {} - } - - current_time = time.time() - week_ago = current_time - (7 * 24 * 60 * 60) - month_ago = current_time - (30 * 24 * 60 * 60) - - tag_counts = {} - - for profile in self.profiles.values(): - # Browser types - browser_type = profile.browser_type - stats['browser_types'][browser_type] = stats['browser_types'].get(browser_type, 0) + 1 - - # Age distribution - if profile.created_at > week_ago: - stats['profiles_by_age']['last_week'] += 1 - elif profile.created_at > month_ago: - stats['profiles_by_age']['last_month'] += 1 - else: - stats['profiles_by_age']['older'] += 1 - - # Tag usage - for tag in profile.tags: - tag_counts[tag] = tag_counts.get(tag, 0) + 1 - - # Sort tags by usage - stats['most_used_tags'] = dict(sorted(tag_counts.items(), key=lambda x: x[1], reverse=True)[:10]) - - return stats - - -# Factory function -def get_browser_profiler(profiles_dir: Optional[Path] = None) -> BrowserProfiler: - """Get browser profiler instance.""" - return BrowserProfiler(profiles_dir) - - -# Convenience functions -def create_browser_profile( - name: str, - browser_type: str = "chromium", - description: str = None, - tags: List[str] = None, - profiles_dir: Optional[Path] = None -) -> str: - """Create a browser profile.""" - profiler = get_browser_profiler(profiles_dir) - return profiler.create_profile(name, browser_type, description, tags) - - -def get_profile_browser_config( - profile_id: str, - profiles_dir: Optional[Path] = None -) -> Optional[BrowserConfig]: - """Get browser config for a profile.""" - profiler = get_browser_profiler(profiles_dir) - return profiler.get_browser_config(profile_id) - - -def list_browser_profiles( - tags: List[str] = None, - profiles_dir: Optional[Path] = None -) -> List[BrowserProfile]: - """List browser profiles.""" - profiler = get_browser_profiler(profiles_dir) - return profiler.list_profiles(tags) diff --git a/apps/backend/app/services/cache.py b/apps/backend/app/services/cache.py deleted file mode 100644 index 99b06e5..0000000 --- a/apps/backend/app/services/cache.py +++ /dev/null @@ -1,433 +0,0 @@ -""" -Redis caching service for search results and content. -Supports both Upstash Redis (REST API) and regular Redis. -""" -import json -import gzip -import hashlib -from typing import Optional, Any, Dict, List, Union -from datetime import datetime, timedelta -import redis.asyncio as redis -from redis.asyncio.connection import ConnectionPool -import orjson - -# Upstash Redis client -try: - from upstash_redis.asyncio import Redis as UpstashRedis - UPSTASH_AVAILABLE = True -except ImportError: - UPSTASH_AVAILABLE = False - UpstashRedis = None - -from app.config import get_settings -from app.models.responses import UnQuestResponse -from app.models.requests import UnQuestRequest -import structlog - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class CacheService: - """Redis-based caching service with compression and multi-layer caching. - - Supports both Upstash Redis (REST API) and regular Redis connections. - Automatically uses Upstash when credentials are available. - """ - - def __init__(self): - self.redis_url = settings.redis_url - self.default_ttl = settings.cache_default_ttl - self.compression_enabled = settings.cache_compression - self._pool: Optional[ConnectionPool] = None - self._client: Optional[Union[redis.Redis, UpstashRedis]] = None - self._use_upstash = ( - UPSTASH_AVAILABLE and - settings.upstash_redis_rest_url and - settings.upstash_redis_rest_token - ) - - async def __aenter__(self): - """Async context manager entry.""" - await self.initialize() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit.""" - await self.close() - - async def initialize(self): - """Initialize Redis connection (Upstash or regular Redis).""" - if not self._client: - if self._use_upstash: - # Use Upstash Redis REST API - logger.info("initializing_upstash_redis", url=settings.upstash_redis_rest_url) - self._client = UpstashRedis( - url=settings.upstash_redis_rest_url, - token=settings.upstash_redis_rest_token - ) - - # Test connection - try: - await self._client.set("connection_test", "ok", ex=5) - test_result = await self._client.get("connection_test") - await self._client.delete("connection_test") - logger.info("upstash_redis_connection_established", test_result=test_result) - except Exception as e: - logger.error("upstash_redis_connection_failed", error=str(e)) - raise - - else: - # Use regular Redis - logger.info("initializing_regular_redis", url=self.redis_url[:50] + "...") - self._pool = ConnectionPool.from_url( - self.redis_url, - max_connections=settings.redis_max_connections, - decode_responses=False # We'll handle encoding/decoding - ) - self._client = redis.Redis(connection_pool=self._pool) - - # Test connection - try: - await self._client.ping() - logger.info("redis_connection_established") - except Exception as e: - logger.error("redis_connection_failed", error=str(e)) - raise - - async def close(self): - """Close Redis connections.""" - if self._client and not self._use_upstash: - # Only regular Redis clients need explicit closing - await self._client.close() - if self._pool: - await self._pool.disconnect() - self._client = None - self._pool = None - - async def get_search_results(self, cache_key: str) -> Optional[UnQuestResponse]: - """ - Retrieve search results from cache. - - Args: - cache_key: Cache key for the search results - - Returns: - Cached UnQuestResponse or None if not found - """ - if not self._client: - await self.initialize() - - try: - # Get from cache - cached_data = await self._client.get(cache_key) - - if not cached_data: - logger.debug("cache_miss", key=cache_key) - return None - - # Decompress if needed - if self.compression_enabled: - try: - cached_data = gzip.decompress(cached_data) - except: - # Data might not be compressed - pass - - # Deserialize - data = orjson.loads(cached_data) - - # Update hit count - await self._increment_hit_count(cache_key) - - # Convert back to response model - response = UnQuestResponse(**data) - response.cached = True - response.cache_key = cache_key - - logger.info("cache_hit", key=cache_key) - return response - - except Exception as e: - logger.error("cache_get_error", key=cache_key, error=str(e)) - return None - - async def set_search_results( - self, - cache_key: str, - data: UnQuestResponse, - ttl: Optional[int] = None - ): - """ - Cache search results with optional compression. - - Args: - cache_key: Cache key for the search results - data: UnQuestResponse to cache - ttl: Time to live in seconds (uses default if not specified) - """ - if not self._client: - await self.initialize() - - try: - # Use provided TTL or default - ttl = ttl or self.default_ttl - - # Serialize data - serialized = orjson.dumps(data.dict()) - - # Compress if enabled - if self.compression_enabled: - original_size = len(serialized) - serialized = gzip.compress(serialized, compresslevel=6) - compressed_size = len(serialized) - compression_ratio = (1 - compressed_size / original_size) * 100 - logger.debug( - "cache_compression", - original_size=original_size, - compressed_size=compressed_size, - compression_ratio=f"{compression_ratio:.1f}%" - ) - - # Set in Redis with TTL - await self._client.setex(cache_key, ttl, serialized) - - # Store metadata - await self._store_cache_metadata(cache_key, data, ttl) - - logger.info("cache_set", key=cache_key, ttl=ttl) - - except Exception as e: - logger.error("cache_set_error", key=cache_key, error=str(e)) - - async def invalidate_pattern(self, pattern: str): - """ - Invalidate cache entries matching a pattern. - - Args: - pattern: Redis pattern (e.g., "search:query:python*") - """ - if not self._client: - await self.initialize() - - try: - # Find matching keys - cursor = 0 - invalidated_count = 0 - - while True: - cursor, keys = await self._client.scan( - cursor, - match=pattern, - count=100 - ) - - if keys: - # Delete in batch - await self._client.delete(*keys) - invalidated_count += len(keys) - - if cursor == 0: - break - - logger.info( - "cache_invalidated_pattern", - pattern=pattern, - invalidated_count=invalidated_count - ) - - except Exception as e: - logger.error("cache_invalidate_error", pattern=pattern, error=str(e)) - - def generate_cache_key(self, request: UnQuestRequest) -> str: - """ - Generate deterministic cache key from request parameters. - - Args: - request: UnQuestRequest object - - Returns: - Cache key string - """ - # Extract relevant fields for cache key - key_data = { - "query": request.query.lower().strip(), - "engines": sorted(request.engines), - "max_results": request.max_results, - "language": request.language, - "safe_search": request.safe_search, - "scrape_content": request.scrape_content, - "include_images": request.include_images, - "include_links": request.include_links - } - - # Add custom selectors if present - if request.scrape_selectors: - key_data["selectors"] = sorted(request.scrape_selectors.items()) - - # Create deterministic string - key_string = orjson.dumps(key_data, option=orjson.OPT_SORT_KEYS).decode() - - # Generate hash - hash_digest = hashlib.sha256(key_string.encode()).hexdigest()[:16] - - # Create readable key - query_slug = request.query[:20].replace(' ', '_').lower() - cache_key = f"search:{query_slug}:{hash_digest}" - - return cache_key - - async def get_cached_url_content(self, url: str) -> Optional[str]: - """ - Get cached content for a specific URL. - - Args: - url: URL to check - - Returns: - Cached content or None - """ - if not self._client: - await self.initialize() - - url_hash = hashlib.sha256(url.encode()).hexdigest()[:16] - cache_key = f"url_content:{url_hash}" - - try: - content = await self._client.get(cache_key) - if content: - if self._use_upstash: - # Upstash returns string values directly - return content - else: - # Regular Redis returns bytes - return content.decode('utf-8') - except: - pass - - return None - - async def set_cached_url_content( - self, - url: str, - content: str, - ttl: int = 86400 # 24 hours default - ): - """ - Cache content for a specific URL. - - Args: - url: URL of the content - content: Content to cache - ttl: Time to live in seconds - """ - if not self._client: - await self.initialize() - - url_hash = hashlib.sha256(url.encode()).hexdigest()[:16] - cache_key = f"url_content:{url_hash}" - - try: - if self._use_upstash: - # Upstash REST API expects string values - await self._client.setex(cache_key, ttl, content) - else: - # Regular Redis can handle bytes - await self._client.setex(cache_key, ttl, content.encode('utf-8')) - except Exception as e: - logger.error("url_cache_set_error", url=url, error=str(e)) - - async def _increment_hit_count(self, cache_key: str): - """Increment cache hit count for analytics.""" - try: - hit_key = f"{cache_key}:hits" - await self._client.incr(hit_key) - except: - pass # Non-critical operation - - async def _store_cache_metadata( - self, - cache_key: str, - data: UnQuestResponse, - ttl: int - ): - """Store cache metadata for monitoring and analytics.""" - try: - metadata = { - "query": data.search_metadata.query, - "engines": data.search_metadata.engines_used, - "results_count": len(data.results), - "created_at": datetime.utcnow().isoformat(), - "expires_at": (datetime.utcnow() + timedelta(seconds=ttl)).isoformat(), - "ttl": ttl - } - - metadata_key = f"{cache_key}:metadata" - await self._client.setex( - metadata_key, - ttl, - orjson.dumps(metadata) - ) - except: - pass # Non-critical operation - - async def get_cache_stats(self) -> Dict[str, Any]: - """Get cache statistics for monitoring.""" - if not self._client: - await self.initialize() - - try: - info = await self._client.info() - - stats = { - "connected": True, - "used_memory": info.get("used_memory_human", "unknown"), - "connected_clients": info.get("connected_clients", 0), - "total_commands_processed": info.get("total_commands_processed", 0), - "keyspace_hits": info.get("keyspace_hits", 0), - "keyspace_misses": info.get("keyspace_misses", 0), - "hit_rate": 0.0 - } - - # Calculate hit rate - total_ops = stats["keyspace_hits"] + stats["keyspace_misses"] - if total_ops > 0: - stats["hit_rate"] = (stats["keyspace_hits"] / total_ops) * 100 - - return stats - - except Exception as e: - logger.error("cache_stats_error", error=str(e)) - return { - "connected": False, - "error": str(e) - } - - async def warmup_cache(self, popular_queries: List[str]): - """ - Warmup cache with popular queries. - - Args: - popular_queries: List of queries to pre-cache - """ - logger.info("cache_warmup_started", queries_count=len(popular_queries)) - - # This would typically trigger actual searches - # Implementation depends on your search service - - logger.info("cache_warmup_completed") - - -# Singleton instance -_cache_service: Optional[CacheService] = None - - -async def get_cache_service() -> CacheService: - """Get or create cache service instance.""" - global _cache_service - - if _cache_service is None: - _cache_service = CacheService() - await _cache_service.initialize() - - return _cache_service diff --git a/apps/backend/app/services/cache_context.py b/apps/backend/app/services/cache_context.py deleted file mode 100644 index 5f0b9ad..0000000 --- a/apps/backend/app/services/cache_context.py +++ /dev/null @@ -1,532 +0,0 @@ -""" -Advanced cache context management system for intelligent caching decisions. - -This module provides sophisticated cache management: -- Multiple cache modes (ENABLED, DISABLED, READ_ONLY, WRITE_ONLY, BYPASS) -- URL type detection and caching rules -- Context-aware caching decisions -- Performance optimization and cache statistics -""" - -from enum import Enum -from typing import Dict, Any, Optional, List, Callable -from dataclasses import dataclass, field -from urllib.parse import urlparse -import time - -import structlog - -logger = structlog.get_logger(__name__) - - -class CacheMode(str, Enum): - """ - Defines the caching behavior for web crawling operations. - - Modes: - - ENABLED: Normal caching behavior (read and write) - - DISABLED: No caching at all - - READ_ONLY: Only read from cache, don't write - - WRITE_ONLY: Only write to cache, don't read - - BYPASS: Bypass cache for this operation - """ - ENABLED = "enabled" - DISABLED = "disabled" - READ_ONLY = "read_only" - WRITE_ONLY = "write_only" - BYPASS = "bypass" - - -class URLType(str, Enum): - """URL type classification for caching decisions.""" - WEB_HTTP = "web_http" - WEB_HTTPS = "web_https" - LOCAL_FILE = "local_file" - RAW_HTML = "raw_html" - DATA_URI = "data_uri" - FTP = "ftp" - UNKNOWN = "unknown" - - -@dataclass -class CacheRule: - """Rule for cache behavior based on URL patterns or types.""" - pattern: str - cache_mode: CacheMode - ttl: Optional[int] = None # Time to live in seconds - priority: int = 0 # Higher priority rules override lower priority - - def matches(self, url: str, url_type: URLType) -> bool: - """Check if this rule matches the given URL.""" - if self.pattern == "*": - return True - elif self.pattern.startswith("type:"): - return url_type.value == self.pattern[5:] - elif self.pattern.startswith("domain:"): - try: - parsed = urlparse(url) - domain = parsed.netloc.lower() - pattern_domain = self.pattern[7:].lower() - return domain == pattern_domain or domain.endswith("." + pattern_domain) - except Exception: - return False - elif self.pattern in url: - return True - else: - return False - - -@dataclass -class CacheStats: - """Statistics for cache operations.""" - total_requests: int = 0 - cache_hits: int = 0 - cache_misses: int = 0 - cache_writes: int = 0 - cache_bypasses: int = 0 - total_time_saved: float = 0.0 # Time saved by cache hits in seconds - - @property - def hit_rate(self) -> float: - """Calculate cache hit rate percentage.""" - if self.total_requests == 0: - return 0.0 - return (self.cache_hits / self.total_requests) * 100 - - @property - def miss_rate(self) -> float: - """Calculate cache miss rate percentage.""" - return 100.0 - self.hit_rate - - @property - def bypass_rate(self) -> float: - """Calculate cache bypass rate percentage.""" - if self.total_requests == 0: - return 0.0 - return (self.cache_bypasses / self.total_requests) * 100 - - -class CacheContext: - """ - Encapsulates cache-related decisions and URL handling. - - This class centralizes all cache-related logic and URL type checking, - making the caching behavior more predictable and maintainable. - """ - - def __init__(self, - url: str, - cache_mode: CacheMode = CacheMode.ENABLED, - always_bypass: bool = False, - custom_rules: List[CacheRule] = None, - default_ttl: Optional[int] = None): - """ - Initialize the CacheContext with the provided URL and cache mode. - - Args: - url: The URL being processed - cache_mode: The cache mode for the current operation - always_bypass: If True, bypasses caching for this operation - custom_rules: Additional cache rules to apply - default_ttl: Default time-to-live for cache entries - """ - self.url = url - self.cache_mode = cache_mode - self.always_bypass = always_bypass - self.custom_rules = custom_rules or [] - self.default_ttl = default_ttl - - # URL analysis - self.url_type = self._classify_url(url) - self.is_cacheable = self._determine_cacheability() - self.parsed_url = self._safe_parse_url(url) - - # Display formatting - self._url_display = self._format_display_url() - - # Apply custom rules - self._effective_cache_mode = self._apply_cache_rules() - - # Performance tracking - self._start_time = time.time() - self._cache_hit = False - - def _classify_url(self, url: str) -> URLType: - """Classify URL type for caching decisions.""" - if url.startswith("https://"): - return URLType.WEB_HTTPS - elif url.startswith("http://"): - return URLType.WEB_HTTP - elif url.startswith("file://"): - return URLType.LOCAL_FILE - elif url.startswith("raw:"): - return URLType.RAW_HTML - elif url.startswith("data:"): - return URLType.DATA_URI - elif url.startswith("ftp://"): - return URLType.FTP - else: - return URLType.UNKNOWN - - def _determine_cacheability(self) -> bool: - """Determine if URL is cacheable based on type and content.""" - # Web URLs are generally cacheable - if self.url_type in [URLType.WEB_HTTP, URLType.WEB_HTTPS]: - return True - - # Local files can be cached - if self.url_type == URLType.LOCAL_FILE: - return True - - # Raw HTML and data URIs are not typically cached - if self.url_type in [URLType.RAW_HTML, URLType.DATA_URI]: - return False - - # FTP and unknown types - conservative approach - return False - - def _safe_parse_url(self, url: str): - """Safely parse URL, handling malformed URLs gracefully.""" - try: - return urlparse(url) - except Exception: - return None - - def _format_display_url(self) -> str: - """Format URL for display purposes.""" - if self.url_type == URLType.RAW_HTML: - return "Raw HTML Content" - elif self.url_type == URLType.DATA_URI: - return "Data URI" - elif len(self.url) > 100: - return self.url[:97] + "..." - else: - return self.url - - def _apply_cache_rules(self) -> CacheMode: - """Apply custom cache rules to determine effective cache mode.""" - if not self.custom_rules: - return self.cache_mode - - # Find matching rules, sorted by priority - matching_rules = [ - rule for rule in self.custom_rules - if rule.matches(self.url, self.url_type) - ] - - if not matching_rules: - return self.cache_mode - - # Apply highest priority rule - highest_priority_rule = max(matching_rules, key=lambda r: r.priority) - return highest_priority_rule.cache_mode - - def should_read(self) -> bool: - """ - Determine if cache should be read based on context. - - Returns: - bool: True if cache should be read, False otherwise - """ - if self.always_bypass or not self.is_cacheable: - return False - - return self._effective_cache_mode in [CacheMode.ENABLED, CacheMode.READ_ONLY] - - def should_write(self) -> bool: - """ - Determine if cache should be written based on context. - - Returns: - bool: True if cache should be written, False otherwise - """ - if self.always_bypass or not self.is_cacheable: - return False - - return self._effective_cache_mode in [CacheMode.ENABLED, CacheMode.WRITE_ONLY] - - def get_cache_key(self) -> str: - """Generate cache key for this URL and context.""" - # Include URL and relevant parameters in cache key - base_key = f"url:{self.url}" - - # Add URL type to key for better organization - base_key += f":type:{self.url_type.value}" - - return base_key - - def get_ttl(self) -> Optional[int]: - """Get time-to-live for cache entry.""" - # Check if any custom rules specify TTL - for rule in self.custom_rules: - if rule.matches(self.url, self.url_type) and rule.ttl is not None: - return rule.ttl - - # Use default TTL - return self.default_ttl - - def mark_cache_hit(self): - """Mark this request as a cache hit.""" - self._cache_hit = True - - def get_performance_metrics(self) -> Dict[str, Any]: - """Get performance metrics for this cache operation.""" - elapsed_time = time.time() - self._start_time - - return { - 'url': self.url, - 'url_type': self.url_type.value, - 'cache_mode': self._effective_cache_mode.value, - 'cache_hit': self._cache_hit, - 'cacheable': self.is_cacheable, - 'elapsed_time': elapsed_time, - 'should_read': self.should_read(), - 'should_write': self.should_write() - } - - @property - def display_url(self) -> str: - """Returns the URL in display format.""" - return self._url_display - - @property - def domain(self) -> Optional[str]: - """Get domain from URL.""" - if self.parsed_url: - return self.parsed_url.netloc.lower() - return None - - @property - def scheme(self) -> Optional[str]: - """Get URL scheme.""" - if self.parsed_url: - return self.parsed_url.scheme.lower() - return None - - -class CacheContextManager: - """ - Manager for cache contexts with global rules and statistics. - - Provides centralized management of cache rules and performance tracking. - """ - - def __init__(self): - """Initialize cache context manager.""" - self.global_rules: List[CacheRule] = [] - self.stats = CacheStats() - self.domain_stats: Dict[str, CacheStats] = {} - - # Default rules - self._setup_default_rules() - - def _setup_default_rules(self): - """Setup default cache rules.""" - # Web URLs should be cached - self.global_rules.append( - CacheRule("type:web_http", CacheMode.ENABLED, ttl=3600, priority=1) - ) - self.global_rules.append( - CacheRule("type:web_https", CacheMode.ENABLED, ttl=3600, priority=1) - ) - - # Raw HTML should not be cached - self.global_rules.append( - CacheRule("type:raw_html", CacheMode.DISABLED, priority=2) - ) - - # Data URIs should not be cached - self.global_rules.append( - CacheRule("type:data_uri", CacheMode.DISABLED, priority=2) - ) - - # Local files can be cached with shorter TTL - self.global_rules.append( - CacheRule("type:local_file", CacheMode.ENABLED, ttl=1800, priority=1) - ) - - def add_rule(self, rule: CacheRule): - """Add a global cache rule.""" - self.global_rules.append(rule) - # Sort rules by priority - self.global_rules.sort(key=lambda r: r.priority, reverse=True) - - def remove_rule(self, pattern: str): - """Remove cache rule by pattern.""" - self.global_rules = [r for r in self.global_rules if r.pattern != pattern] - - def create_context(self, - url: str, - cache_mode: CacheMode = CacheMode.ENABLED, - **kwargs) -> CacheContext: - """ - Create cache context with global rules applied. - - Args: - url: URL to create context for - cache_mode: Base cache mode - **kwargs: Additional context parameters - - Returns: - CacheContext with global rules applied - """ - # Combine custom rules with global rules - custom_rules = kwargs.get('custom_rules', []) - all_rules = self.global_rules + custom_rules - kwargs['custom_rules'] = all_rules - - context = CacheContext(url, cache_mode, **kwargs) - - # Update statistics - self.stats.total_requests += 1 - - # Track domain-specific stats - if context.domain: - if context.domain not in self.domain_stats: - self.domain_stats[context.domain] = CacheStats() - self.domain_stats[context.domain].total_requests += 1 - - return context - - def record_cache_hit(self, context: CacheContext, time_saved: float = 0.0): - """Record a cache hit.""" - context.mark_cache_hit() - self.stats.cache_hits += 1 - self.stats.total_time_saved += time_saved - - if context.domain and context.domain in self.domain_stats: - self.domain_stats[context.domain].cache_hits += 1 - - def record_cache_miss(self, context: CacheContext): - """Record a cache miss.""" - self.stats.cache_misses += 1 - - if context.domain and context.domain in self.domain_stats: - self.domain_stats[context.domain].cache_misses += 1 - - def record_cache_write(self, context: CacheContext): - """Record a cache write.""" - self.stats.cache_writes += 1 - - if context.domain and context.domain in self.domain_stats: - self.domain_stats[context.domain].cache_writes += 1 - - def record_cache_bypass(self, context: CacheContext): - """Record a cache bypass.""" - self.stats.cache_bypasses += 1 - - if context.domain and context.domain in self.domain_stats: - self.domain_stats[context.domain].cache_bypasses += 1 - - def get_global_stats(self) -> CacheStats: - """Get global cache statistics.""" - return self.stats - - def get_domain_stats(self, domain: str) -> Optional[CacheStats]: - """Get cache statistics for specific domain.""" - return self.domain_stats.get(domain) - - def get_top_domains(self, limit: int = 10) -> List[tuple]: - """Get top domains by request count.""" - domain_counts = [ - (domain, stats.total_requests) - for domain, stats in self.domain_stats.items() - ] - domain_counts.sort(key=lambda x: x[1], reverse=True) - return domain_counts[:limit] - - def reset_stats(self): - """Reset all statistics.""" - self.stats = CacheStats() - self.domain_stats.clear() - - def export_rules(self) -> List[Dict[str, Any]]: - """Export cache rules to dictionary format.""" - return [ - { - 'pattern': rule.pattern, - 'cache_mode': rule.cache_mode.value, - 'ttl': rule.ttl, - 'priority': rule.priority - } - for rule in self.global_rules - ] - - def import_rules(self, rules_data: List[Dict[str, Any]]): - """Import cache rules from dictionary format.""" - self.global_rules.clear() - - for rule_data in rules_data: - rule = CacheRule( - pattern=rule_data['pattern'], - cache_mode=CacheMode(rule_data['cache_mode']), - ttl=rule_data.get('ttl'), - priority=rule_data.get('priority', 0) - ) - self.global_rules.append(rule) - - # Sort by priority - self.global_rules.sort(key=lambda r: r.priority, reverse=True) - - -# Singleton instance for global cache management -_cache_manager: Optional[CacheContextManager] = None - - -def get_cache_manager() -> CacheContextManager: - """Get singleton cache context manager.""" - global _cache_manager - if _cache_manager is None: - _cache_manager = CacheContextManager() - return _cache_manager - - -# Legacy compatibility functions -def _legacy_to_cache_mode(disable_cache: bool = False, - bypass_cache: bool = False, - no_cache_read: bool = False, - no_cache_write: bool = False) -> CacheMode: - """Convert legacy cache parameters to CacheMode.""" - if disable_cache: - return CacheMode.DISABLED - elif bypass_cache: - return CacheMode.BYPASS - elif no_cache_read and no_cache_write: - return CacheMode.DISABLED - elif no_cache_read: - return CacheMode.WRITE_ONLY - elif no_cache_write: - return CacheMode.READ_ONLY - else: - return CacheMode.ENABLED - - -# Convenience functions -def create_cache_context(url: str, **legacy_params) -> CacheContext: - """ - Create cache context with legacy parameter support. - - Args: - url: URL to create context for - **legacy_params: Legacy cache parameters for backward compatibility - - Returns: - CacheContext instance - """ - # Extract cache mode from legacy parameters - cache_mode = _legacy_to_cache_mode( - disable_cache=legacy_params.get('disable_cache', False), - bypass_cache=legacy_params.get('bypass_cache', False), - no_cache_read=legacy_params.get('no_cache_read', False), - no_cache_write=legacy_params.get('no_cache_write', False) - ) - - # Use cache manager for consistency - manager = get_cache_manager() - return manager.create_context(url, cache_mode) - - -def should_cache_url(url: str) -> bool: - """Quick check if URL should be cached.""" - context = create_cache_context(url) - return context.should_read() or context.should_write() diff --git a/apps/backend/app/services/change_tracking.py b/apps/backend/app/services/change_tracking.py deleted file mode 100644 index 484d57a..0000000 --- a/apps/backend/app/services/change_tracking.py +++ /dev/null @@ -1,709 +0,0 @@ -""" -Website change tracking service inspired by Firecrawl. - -Provides comprehensive change monitoring capabilities: -- Content comparison and diff generation -- Change detection algorithms -- Historical data storage -- Notification systems -- Advanced diff visualization -""" - -import asyncio -import time -import hashlib -import json -import difflib -from typing import Dict, List, Optional, Any, Union, Literal -from dataclasses import dataclass, field -from datetime import datetime, timedelta -from enum import Enum -import structlog - -from app.config import get_settings -from app.models.responses import ScrapedContent, ContentMetadata -from app.services.enhanced_scraping import get_enhanced_scraping_service -from app.services.database import get_database_service - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class ChangeStatus(Enum): - """Status of content changes.""" - NEW = "new" - SAME = "same" - CHANGED = "changed" - REMOVED = "removed" - - -class VisibilityStatus(Enum): - """Visibility status of content.""" - VISIBLE = "visible" - HIDDEN = "hidden" - - -@dataclass -class ContentChange: - """Represents a change in content.""" - change_type: str # "add", "delete", "modify" - content: str - line_number: Optional[int] = None - position: Optional[int] = None - normal: bool = True - - -@dataclass -class DiffChunk: - """A chunk of diff information.""" - content: str - changes: List[ContentChange] - - -@dataclass -class DiffFile: - """File-level diff information.""" - from_version: Optional[str] = None - to_version: Optional[str] = None - chunks: List[DiffChunk] = field(default_factory=list) - - -@dataclass -class ContentDiff: - """Comprehensive diff result.""" - text_diff: str - json_diff: Dict[str, Any] - - -@dataclass -class ChangeTrackingData: - """Complete change tracking information.""" - previous_scrape_at: Optional[datetime] = None - change_status: ChangeStatus = ChangeStatus.NEW - visibility: VisibilityStatus = VisibilityStatus.VISIBLE - diff: Optional[ContentDiff] = None - previous_content_hash: Optional[str] = None - current_content_hash: Optional[str] = None - change_percentage: float = 0.0 - significant_changes: List[str] = field(default_factory=list) - - -@dataclass -class ChangeTrackingConfig: - """Configuration for change tracking.""" - enabled: bool = True - tag: Optional[str] = None # Tag for grouping tracked content - threshold: float = 0.05 # Minimum change percentage to trigger notification - compare_text: bool = True - compare_html: bool = True - compare_metadata: bool = True - store_history: bool = True - max_history_entries: int = 100 - notification_webhook: Optional[str] = None - diff_format: Literal["text", "html", "json"] = "text" - - -@dataclass -class TrackedContent: - """Stored content for tracking.""" - url: str - content_hash: str - content_data: Dict[str, Any] # Serialized content - scraped_at: datetime - tag: Optional[str] = None - metadata: Dict[str, Any] = field(default_factory=dict) - - -@dataclass -class ChangeTrackingResult: - """Result of change tracking operation.""" - url: str - tracking_data: ChangeTrackingData - scraped_content: ScrapedContent - processing_time_ms: int - success: bool - error: Optional[str] = None - - -class ContentHasher: - """Handles content hashing for change detection.""" - - @staticmethod - def hash_content(content: str) -> str: - """Generate hash of content.""" - return hashlib.sha256(content.encode('utf-8')).hexdigest() - - @staticmethod - def hash_scraped_content(scraped_content: ScrapedContent) -> str: - """Generate hash of scraped content.""" - # Combine relevant fields for hashing - content_parts = [ - scraped_content.text or "", - scraped_content.html or "", - scraped_content.title or "", - json.dumps(scraped_content.metadata.dict() if scraped_content.metadata else {}, sort_keys=True) - ] - - combined_content = "\n---SEPARATOR---\n".join(content_parts) - return ContentHasher.hash_content(combined_content) - - @staticmethod - def hash_specific_fields(data: Dict[str, Any], fields: List[str]) -> str: - """Hash specific fields of data.""" - field_data = {field: data.get(field, "") for field in fields} - content = json.dumps(field_data, sort_keys=True) - return ContentHasher.hash_content(content) - - -class ContentDiffer: - """Generates diffs between content versions.""" - - def generate_text_diff(self, old_text: str, new_text: str) -> str: - """Generate text-based diff.""" - old_lines = old_text.splitlines(keepends=True) - new_lines = new_text.splitlines(keepends=True) - - diff = difflib.unified_diff( - old_lines, - new_lines, - fromfile='previous', - tofile='current', - lineterm='' - ) - - return ''.join(diff) - - def generate_html_diff(self, old_text: str, new_text: str) -> str: - """Generate HTML-based diff.""" - differ = difflib.HtmlDiff() - return differ.make_file( - old_text.splitlines(), - new_text.splitlines(), - fromdesc='Previous Version', - todesc='Current Version' - ) - - def generate_json_diff(self, old_content: Dict[str, Any], new_content: Dict[str, Any]) -> Dict[str, Any]: - """Generate JSON-based structural diff.""" - files = [] - - # Compare text content - if old_content.get("text") != new_content.get("text"): - text_changes = self._analyze_text_changes( - old_content.get("text", ""), - new_content.get("text", "") - ) - - files.append({ - "from": "text", - "to": "text", - "chunks": text_changes - }) - - # Compare HTML content - if old_content.get("html") != new_content.get("html"): - html_changes = self._analyze_text_changes( - old_content.get("html", ""), - new_content.get("html", "") - ) - - files.append({ - "from": "html", - "to": "html", - "chunks": html_changes - }) - - return {"files": files} - - def _analyze_text_changes(self, old_text: str, new_text: str) -> List[Dict[str, Any]]: - """Analyze changes between two text versions.""" - changes = [] - - # Use difflib to get detailed changes - old_lines = old_text.splitlines() - new_lines = new_text.splitlines() - - matcher = difflib.SequenceMatcher(None, old_lines, new_lines) - - for tag, i1, i2, j1, j2 in matcher.get_opcodes(): - if tag == 'equal': - continue - - chunk_changes = [] - - if tag == 'delete': - for line_num in range(i1, i2): - chunk_changes.append({ - "type": "delete", - "ln": line_num + 1, - "content": old_lines[line_num] if line_num < len(old_lines) else "", - "normal": False - }) - - elif tag == 'insert': - for line_num in range(j1, j2): - chunk_changes.append({ - "type": "add", - "ln": line_num + 1, - "content": new_lines[line_num] if line_num < len(new_lines) else "", - "normal": False - }) - - elif tag == 'replace': - # Handle replacements - for line_num in range(i1, i2): - chunk_changes.append({ - "type": "delete", - "ln1": line_num + 1, - "content": old_lines[line_num] if line_num < len(old_lines) else "", - "normal": False - }) - - for line_num in range(j1, j2): - chunk_changes.append({ - "type": "add", - "ln2": line_num + 1, - "content": new_lines[line_num] if line_num < len(new_lines) else "", - "normal": False - }) - - if chunk_changes: - changes.append({ - "content": f"Lines {i1+1}-{i2} / {j1+1}-{j2}", - "changes": chunk_changes - }) - - return changes - - def calculate_change_percentage(self, old_content: str, new_content: str) -> float: - """Calculate percentage of content that changed.""" - if not old_content and not new_content: - return 0.0 - - if not old_content: - return 100.0 - - if not new_content: - return 100.0 - - # Use difflib to calculate similarity - similarity = difflib.SequenceMatcher(None, old_content, new_content).ratio() - return (1 - similarity) * 100 - - -class ChangeTrackingService: - """ - Website change tracking service. - - Provides comprehensive change monitoring including: - - Content comparison and diff generation - - Historical data storage - - Change notifications - - Advanced analytics - """ - - def __init__(self): - """Initialize change tracking service.""" - self.content_hasher = ContentHasher() - self.content_differ = ContentDiffer() - self.tracked_content_cache: Dict[str, TrackedContent] = {} - self.tracking_stats = { - "total_tracked": 0, - "changes_detected": 0, - "notifications_sent": 0 - } - - async def track_content_changes( - self, - url: str, - config: Optional[ChangeTrackingConfig] = None - ) -> ChangeTrackingResult: - """ - Track changes for a specific URL. - - Args: - url: URL to track - config: Change tracking configuration - - Returns: - ChangeTrackingResult with change information - """ - start_time = time.time() - config = config or ChangeTrackingConfig() - - self.tracking_stats["total_tracked"] += 1 - - logger.info("change_tracking_started", url=url, tag=config.tag) - - try: - # Scrape current content - scraping_service = await get_enhanced_scraping_service() - current_results = await scraping_service.scrape_urls_enhanced([url]) - - if not current_results or not current_results[0].extraction_success: - raise ValueError("Failed to scrape current content") - - current_content = current_results[0] - - # Get previous content - previous_content = await self._get_previous_content(url, config.tag) - - # Generate content hash - current_hash = self.content_hasher.hash_scraped_content(current_content) - - # Determine change status - if previous_content is None: - change_status = ChangeStatus.NEW - change_tracking_data = ChangeTrackingData( - change_status=change_status, - current_content_hash=current_hash - ) - else: - change_status = ( - ChangeStatus.SAME if previous_content.content_hash == current_hash - else ChangeStatus.CHANGED - ) - - # Generate diff if content changed - diff = None - change_percentage = 0.0 - significant_changes = [] - - if change_status == ChangeStatus.CHANGED: - diff = await self._generate_content_diff( - previous_content, current_content, config - ) - - change_percentage = self.content_differ.calculate_change_percentage( - previous_content.content_data.get("text", ""), - current_content.text or "" - ) - - significant_changes = self._detect_significant_changes( - previous_content, current_content - ) - - self.tracking_stats["changes_detected"] += 1 - - change_tracking_data = ChangeTrackingData( - previous_scrape_at=previous_content.scraped_at, - change_status=change_status, - diff=diff, - previous_content_hash=previous_content.content_hash, - current_content_hash=current_hash, - change_percentage=change_percentage, - significant_changes=significant_changes - ) - - # Store current content for future tracking - if config.store_history: - await self._store_content(url, current_content, config) - - # Send notifications if needed - if (change_status == ChangeStatus.CHANGED and - config.notification_webhook and - change_tracking_data.change_percentage >= config.threshold): - - await self._send_change_notification( - url, change_tracking_data, config - ) - self.tracking_stats["notifications_sent"] += 1 - - # Add change tracking data to scraped content - current_content_dict = current_content.dict() - current_content_dict["changeTracking"] = { - "previousScrapeAt": change_tracking_data.previous_scrape_at.isoformat() if change_tracking_data.previous_scrape_at else None, - "changeStatus": change_tracking_data.change_status.value, - "visibility": change_tracking_data.visibility.value, - "diff": { - "text": change_tracking_data.diff.text_diff if change_tracking_data.diff else "", - "json": change_tracking_data.diff.json_diff if change_tracking_data.diff else {} - } if change_tracking_data.diff else None - } - - enhanced_content = ScrapedContent(**current_content_dict) - - processing_time_ms = int((time.time() - start_time) * 1000) - - result = ChangeTrackingResult( - url=url, - tracking_data=change_tracking_data, - scraped_content=enhanced_content, - processing_time_ms=processing_time_ms, - success=True - ) - - logger.info("change_tracking_completed", - url=url, - change_status=change_status.value, - change_percentage=change_tracking_data.change_percentage, - processing_time_ms=processing_time_ms) - - return result - - except Exception as e: - processing_time_ms = int((time.time() - start_time) * 1000) - error_msg = str(e) - - logger.error("change_tracking_failed", - url=url, - error=error_msg, - processing_time_ms=processing_time_ms) - - return ChangeTrackingResult( - url=url, - tracking_data=ChangeTrackingData(), - scraped_content=ScrapedContent(url=url, extraction_success=False, text=""), - processing_time_ms=processing_time_ms, - success=False, - error=error_msg - ) - - async def _get_previous_content( - self, - url: str, - tag: Optional[str] = None - ) -> Optional[TrackedContent]: - """Retrieve previously stored content for URL.""" - try: - # Check cache first - cache_key = f"{url}:{tag or 'default'}" - if cache_key in self.tracked_content_cache: - return self.tracked_content_cache[cache_key] - - # In a real implementation, this would query the database - # For now, return None (indicating no previous content) - return None - - except Exception as e: - logger.error("get_previous_content_failed", url=url, error=str(e)) - return None - - async def _store_content( - self, - url: str, - content: ScrapedContent, - config: ChangeTrackingConfig - ): - """Store content for future change tracking.""" - try: - content_hash = self.content_hasher.hash_scraped_content(content) - - tracked_content = TrackedContent( - url=url, - content_hash=content_hash, - content_data={ - "text": content.text, - "html": content.html, - "title": content.title, - "metadata": content.metadata.dict() if content.metadata else {} - }, - scraped_at=datetime.utcnow(), - tag=config.tag, - metadata={ - "word_count": content.word_count, - "language": content.language_detected, - "quality_score": content.content_quality_score - } - ) - - # Store in cache - cache_key = f"{url}:{config.tag or 'default'}" - self.tracked_content_cache[cache_key] = tracked_content - - # In a real implementation, this would store in database - logger.debug("content_stored", url=url, tag=config.tag, content_hash=content_hash[:8]) - - except Exception as e: - logger.error("store_content_failed", url=url, error=str(e)) - - async def _generate_content_diff( - self, - previous_content: TrackedContent, - current_content: ScrapedContent, - config: ChangeTrackingConfig - ) -> ContentDiff: - """Generate comprehensive diff between content versions.""" - try: - old_data = previous_content.content_data - new_data = { - "text": current_content.text or "", - "html": current_content.html or "", - "title": current_content.title or "", - "metadata": current_content.metadata.dict() if current_content.metadata else {} - } - - # Generate text diff - text_diff = "" - if config.compare_text: - text_diff = self.content_differ.generate_text_diff( - old_data.get("text", ""), - new_data.get("text", "") - ) - - # Generate JSON diff - json_diff = {} - if config.diff_format == "json": - json_diff = self.content_differ.generate_json_diff(old_data, new_data) - - return ContentDiff( - text_diff=text_diff, - json_diff=json_diff - ) - - except Exception as e: - logger.error("generate_diff_failed", error=str(e)) - return ContentDiff(text_diff="", json_diff={}) - - def _detect_significant_changes( - self, - previous_content: TrackedContent, - current_content: ScrapedContent - ) -> List[str]: - """Detect significant changes between content versions.""" - changes = [] - - old_data = previous_content.content_data - - # Check title changes - if old_data.get("title") != current_content.title: - changes.append("title_changed") - - # Check significant text changes (more than 20% change) - old_text = old_data.get("text", "") - new_text = current_content.text or "" - - if old_text and new_text: - change_pct = self.content_differ.calculate_change_percentage(old_text, new_text) - if change_pct > 20: - changes.append(f"major_text_change_{change_pct:.1f}%") - elif change_pct > 5: - changes.append(f"minor_text_change_{change_pct:.1f}%") - - # Check word count changes - old_word_count = len(old_text.split()) if old_text else 0 - new_word_count = current_content.word_count or 0 - - if abs(old_word_count - new_word_count) > max(10, old_word_count * 0.1): - changes.append("word_count_change") - - return changes - - async def _send_change_notification( - self, - url: str, - tracking_data: ChangeTrackingData, - config: ChangeTrackingConfig - ): - """Send webhook notification about changes.""" - try: - if not config.notification_webhook: - return - - notification_data = { - "url": url, - "timestamp": datetime.utcnow().isoformat(), - "change_status": tracking_data.change_status.value, - "change_percentage": tracking_data.change_percentage, - "significant_changes": tracking_data.significant_changes, - "tag": config.tag, - "previous_scrape": tracking_data.previous_scrape_at.isoformat() if tracking_data.previous_scrape_at else None - } - - # Send webhook (simplified implementation) - import httpx - async with httpx.AsyncClient() as client: - response = await client.post( - config.notification_webhook, - json=notification_data, - timeout=10 - ) - - if response.status_code == 200: - logger.info("change_notification_sent", url=url, webhook=config.notification_webhook) - else: - logger.warning("change_notification_failed", - url=url, - status_code=response.status_code) - - except Exception as e: - logger.error("send_notification_failed", url=url, error=str(e)) - - async def get_change_history( - self, - url: str, - tag: Optional[str] = None, - limit: int = 10 - ) -> List[TrackedContent]: - """Get change history for a URL.""" - try: - # In a real implementation, this would query the database - # For now, return cached content if available - cache_key = f"{url}:{tag or 'default'}" - if cache_key in self.tracked_content_cache: - return [self.tracked_content_cache[cache_key]] - - return [] - - except Exception as e: - logger.error("get_change_history_failed", url=url, error=str(e)) - return [] - - async def get_tracking_stats(self) -> Dict[str, Any]: - """Get change tracking statistics.""" - return { - "tracking_stats": self.tracking_stats, - "cached_content": len(self.tracked_content_cache), - "change_detection_rate": ( - self.tracking_stats["changes_detected"] / - max(1, self.tracking_stats["total_tracked"]) - ) if self.tracking_stats["total_tracked"] > 0 else 0, - "notification_rate": ( - self.tracking_stats["notifications_sent"] / - max(1, self.tracking_stats["changes_detected"]) - ) if self.tracking_stats["changes_detected"] > 0 else 0 - } - - -# Singleton service -_change_tracking_service: Optional[ChangeTrackingService] = None - - -async def get_change_tracking_service() -> ChangeTrackingService: - """Get or create change tracking service instance.""" - global _change_tracking_service - - if _change_tracking_service is None: - _change_tracking_service = ChangeTrackingService() - - return _change_tracking_service - - -# Convenience function -async def track_url_changes( - url: str, - tag: Optional[str] = None, - threshold: float = 0.05, - webhook_url: Optional[str] = None -) -> ChangeTrackingResult: - """ - Convenience function for tracking URL changes. - - Args: - url: URL to track - tag: Optional tag for grouping - threshold: Minimum change percentage to trigger notification - webhook_url: Optional webhook for notifications - - Returns: - ChangeTrackingResult with tracking information - """ - service = await get_change_tracking_service() - - config = ChangeTrackingConfig( - tag=tag, - threshold=threshold, - notification_webhook=webhook_url - ) - - return await service.track_content_changes(url, config) diff --git a/apps/backend/app/services/chunking_strategies.py b/apps/backend/app/services/chunking_strategies.py deleted file mode 100644 index 201cd35..0000000 --- a/apps/backend/app/services/chunking_strategies.py +++ /dev/null @@ -1,593 +0,0 @@ -""" -Text chunking strategies for breaking down content into manageable pieces. - -This module provides various strategies for chunking text content: -- RegexChunking: Split text using regular expressions -- SentenceChunking: Split by sentences using NLP -- TopicChunking: Segment by topics using statistical methods -- FixedSizeChunking: Split into fixed-size chunks -""" - -import re -import math -from abc import ABC, abstractmethod -from typing import List, Dict, Any, Optional, Union -from collections import Counter - -import structlog - -logger = structlog.get_logger(__name__) - - -class ChunkingStrategy(ABC): - """Abstract base class for chunking strategies.""" - - @abstractmethod - def chunk(self, text: str, **kwargs) -> List[str]: - """ - Chunk the given text into smaller pieces. - - Args: - text: The text to chunk - **kwargs: Additional parameters for chunking - - Returns: - List of text chunks - """ - pass - - -class IdentityChunking(ChunkingStrategy): - """Chunking strategy that returns the input text as a single chunk.""" - - def chunk(self, text: str, **kwargs) -> List[str]: - """Return text as single chunk.""" - return [text] if text.strip() else [] - - -class RegexChunking(ChunkingStrategy): - """ - Chunking strategy that splits text based on regular expression patterns. - - Supports multiple patterns that are applied sequentially to split text - into increasingly smaller chunks. - """ - - def __init__(self, patterns: Optional[List[str]] = None, **kwargs): - """ - Initialize regex chunking strategy. - - Args: - patterns: List of regex patterns to split text - Default: [r"\n\n", r"\n", r"\. "] - """ - self.patterns = patterns or [r"\n\n", r"\n", r"\. "] - self.min_chunk_length = kwargs.get('min_chunk_length', 10) - self.max_chunk_length = kwargs.get('max_chunk_length', 5000) - - def chunk(self, text: str, **kwargs) -> List[str]: - """Split text using regex patterns.""" - if not text.strip(): - return [] - - chunks = [text] - - # Apply each pattern sequentially - for pattern in self.patterns: - new_chunks = [] - for chunk in chunks: - if len(chunk) > self.max_chunk_length: - # Split this chunk further - split_chunks = re.split(pattern, chunk) - new_chunks.extend([c.strip() for c in split_chunks if c.strip()]) - else: - new_chunks.append(chunk) - chunks = new_chunks - - # Filter by minimum length - filtered_chunks = [ - chunk for chunk in chunks - if len(chunk.strip()) >= self.min_chunk_length - ] - - return filtered_chunks - - -class SentenceChunking(ChunkingStrategy): - """ - Chunking strategy that splits text into sentences. - - Uses regex patterns to identify sentence boundaries, with support - for common abbreviations and edge cases. - """ - - def __init__(self, **kwargs): - """Initialize sentence chunking strategy.""" - self.min_sentence_length = kwargs.get('min_sentence_length', 10) - self.merge_short_sentences = kwargs.get('merge_short_sentences', True) - - # Regex pattern for sentence splitting - self.sentence_pattern = re.compile( - r'(? List[str]: - """Split text into sentences.""" - if not text.strip(): - return [] - - # Split by sentence boundaries - sentences = self.sentence_pattern.split(text) - sentences = [s.strip() for s in sentences if s.strip()] - - # Optionally merge short sentences - if self.merge_short_sentences: - sentences = self._merge_short_sentences(sentences) - - # Filter by minimum length - return [s for s in sentences if len(s) >= self.min_sentence_length] - - def _merge_short_sentences(self, sentences: List[str]) -> List[str]: - """Merge sentences that are too short with adjacent ones.""" - if not sentences: - return [] - - merged = [] - current = sentences[0] - - for i in range(1, len(sentences)): - if len(current) < self.min_sentence_length * 2: - # Merge with next sentence - current += " " + sentences[i] - else: - merged.append(current) - current = sentences[i] - - # Add the last sentence - if current: - merged.append(current) - - return merged - - -class ParagraphChunking(ChunkingStrategy): - """ - Chunking strategy that splits text into paragraphs. - - Uses double newlines and other paragraph indicators to split text - while preserving semantic boundaries. - """ - - def __init__(self, **kwargs): - """Initialize paragraph chunking strategy.""" - self.min_paragraph_length = kwargs.get('min_paragraph_length', 50) - self.max_paragraph_length = kwargs.get('max_paragraph_length', 2000) - - def chunk(self, text: str, **kwargs) -> List[str]: - """Split text into paragraphs.""" - if not text.strip(): - return [] - - # Split by paragraph indicators - paragraphs = re.split(r'\n\s*\n', text) - paragraphs = [p.strip() for p in paragraphs if p.strip()] - - # Handle overly long paragraphs - processed_paragraphs = [] - for para in paragraphs: - if len(para) > self.max_paragraph_length: - # Split long paragraphs by sentences - sentence_chunker = SentenceChunking( - min_sentence_length=self.min_paragraph_length // 2 - ) - sub_chunks = sentence_chunker.chunk(para) - - # Group sentences into paragraph-sized chunks - current_chunk = "" - for sentence in sub_chunks: - if len(current_chunk + sentence) <= self.max_paragraph_length: - current_chunk += " " + sentence if current_chunk else sentence - else: - if current_chunk: - processed_paragraphs.append(current_chunk) - current_chunk = sentence - - if current_chunk: - processed_paragraphs.append(current_chunk) - else: - processed_paragraphs.append(para) - - # Filter by minimum length - return [p for p in processed_paragraphs if len(p) >= self.min_paragraph_length] - - -class FixedSizeChunking(ChunkingStrategy): - """ - Chunking strategy that splits text into fixed-size chunks. - - Useful for handling token limits in LLMs or creating uniform - chunks for processing. - """ - - def __init__(self, chunk_size: int = 1000, overlap: int = 100, **kwargs): - """ - Initialize fixed-size chunking strategy. - - Args: - chunk_size: Target size for each chunk in characters - overlap: Number of characters to overlap between chunks - """ - self.chunk_size = chunk_size - self.overlap = overlap - self.preserve_words = kwargs.get('preserve_words', True) - - def chunk(self, text: str, **kwargs) -> List[str]: - """Split text into fixed-size chunks.""" - if not text.strip(): - return [] - - if len(text) <= self.chunk_size: - return [text] - - chunks = [] - start = 0 - - while start < len(text): - # Calculate end position - end = min(start + self.chunk_size, len(text)) - - # If preserving words, adjust boundaries - if self.preserve_words and end < len(text): - # Find the last space before the cut-off - space_pos = text.rfind(' ', start, end) - if space_pos > start: - end = space_pos - - chunk = text[start:end].strip() - if chunk: - chunks.append(chunk) - - # Move start position with overlap - start = end - self.overlap if end - self.overlap > start else end - - return chunks - - -class TopicChunking(ChunkingStrategy): - """ - Advanced chunking strategy that attempts to segment text by topics. - - Uses statistical methods to identify topic boundaries and create - semantically coherent chunks. - """ - - def __init__(self, window_size: int = 3, k: int = 10, **kwargs): - """ - Initialize topic chunking strategy. - - Args: - window_size: Size of sliding window for coherence calculation - k: Number of top sentences to consider per window - """ - self.window_size = window_size - self.k = k - self.min_chunk_length = kwargs.get('min_chunk_length', 100) - self.similarity_threshold = kwargs.get('similarity_threshold', 0.3) - - def chunk(self, text: str, **kwargs) -> List[str]: - """Split text into topic-based chunks.""" - if not text.strip(): - return [] - - # First split into sentences - sentence_chunker = SentenceChunking(min_sentence_length=20) - sentences = sentence_chunker.chunk(text) - - if len(sentences) <= self.window_size: - return [text] # Too few sentences for topic segmentation - - # Calculate coherence scores between adjacent windows - boundaries = self._find_topic_boundaries(sentences) - - # Create chunks based on boundaries - chunks = [] - start_idx = 0 - - for boundary in boundaries: - chunk_sentences = sentences[start_idx:boundary] - chunk_text = ' '.join(chunk_sentences) - - if len(chunk_text) >= self.min_chunk_length: - chunks.append(chunk_text) - elif chunks: - # Merge with previous chunk if too small - chunks[-1] += ' ' + chunk_text - else: - # First chunk is small, keep it anyway - chunks.append(chunk_text) - - start_idx = boundary - - # Handle remaining sentences - if start_idx < len(sentences): - remaining = ' '.join(sentences[start_idx:]) - if remaining and len(remaining) >= self.min_chunk_length: - chunks.append(remaining) - elif chunks: - chunks[-1] += ' ' + remaining - - return [c for c in chunks if c.strip()] - - def _find_topic_boundaries(self, sentences: List[str]) -> List[int]: - """Find topic boundaries using coherence analysis.""" - if len(sentences) <= self.window_size * 2: - return [len(sentences)] # Not enough sentences for analysis - - # Calculate vocabulary for each window - window_vocabs = [] - for i in range(len(sentences) - self.window_size + 1): - window_text = ' '.join(sentences[i:i + self.window_size]) - vocab = self._extract_vocabulary(window_text) - window_vocabs.append(vocab) - - # Calculate similarity between adjacent windows - similarities = [] - for i in range(len(window_vocabs) - 1): - sim = self._calculate_similarity(window_vocabs[i], window_vocabs[i + 1]) - similarities.append(sim) - - # Find local minima as topic boundaries - boundaries = [] - for i in range(1, len(similarities) - 1): - if (similarities[i] < similarities[i-1] and - similarities[i] < similarities[i+1] and - similarities[i] < self.similarity_threshold): - # Boundary is at the end of the current window - boundary_idx = i + self.window_size - boundaries.append(boundary_idx) - - # Always add the end as a boundary - boundaries.append(len(sentences)) - - return boundaries - - def _extract_vocabulary(self, text: str) -> Dict[str, int]: - """Extract vocabulary from text with basic preprocessing.""" - # Simple word extraction and counting - words = re.findall(r'\b\w+\b', text.lower()) - # Filter out very short words and common stop words - stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'} - words = [w for w in words if len(w) > 2 and w not in stop_words] - return Counter(words) - - def _calculate_similarity(self, vocab1: Dict[str, int], vocab2: Dict[str, int]) -> float: - """Calculate cosine similarity between two vocabulary distributions.""" - # Get all unique words - all_words = set(vocab1.keys()) | set(vocab2.keys()) - - if not all_words: - return 0.0 - - # Create vectors - vec1 = [vocab1.get(word, 0) for word in all_words] - vec2 = [vocab2.get(word, 0) for word in all_words] - - # Calculate cosine similarity - dot_product = sum(a * b for a, b in zip(vec1, vec2)) - norm1 = math.sqrt(sum(a * a for a in vec1)) - norm2 = math.sqrt(sum(b * b for b in vec2)) - - if norm1 == 0 or norm2 == 0: - return 0.0 - - return dot_product / (norm1 * norm2) - - -class HybridChunking(ChunkingStrategy): - """ - Hybrid chunking strategy that combines multiple approaches. - - Uses a cascade of chunking strategies to create optimal chunks - that balance semantic coherence with size constraints. - """ - - def __init__( - self, - primary_strategy: str = 'paragraph', - fallback_strategy: str = 'sentence', - target_size: int = 1000, - max_size: int = 2000, - min_size: int = 100, - **kwargs - ): - """ - Initialize hybrid chunking strategy. - - Args: - primary_strategy: Primary strategy to use ('paragraph', 'topic', 'sentence') - fallback_strategy: Fallback if primary produces unsuitable chunks - target_size: Target size for chunks - max_size: Maximum allowable chunk size - min_size: Minimum allowable chunk size - """ - self.primary_strategy = primary_strategy - self.fallback_strategy = fallback_strategy - self.target_size = target_size - self.max_size = max_size - self.min_size = min_size - - # Initialize strategies - self.strategies = { - 'paragraph': ParagraphChunking( - min_paragraph_length=min_size, - max_paragraph_length=max_size - ), - 'topic': TopicChunking( - min_chunk_length=min_size - ), - 'sentence': SentenceChunking( - min_sentence_length=min_size // 4 - ), - 'fixed': FixedSizeChunking( - chunk_size=target_size, - overlap=target_size // 10 - ) - } - - def chunk(self, text: str, **kwargs) -> List[str]: - """Apply hybrid chunking strategy.""" - if not text.strip(): - return [] - - # Try primary strategy - primary_chunks = self.strategies[self.primary_strategy].chunk(text) - - # Evaluate chunks and refine if needed - refined_chunks = [] - - for chunk in primary_chunks: - if self.min_size <= len(chunk) <= self.max_size: - # Chunk is good size - refined_chunks.append(chunk) - elif len(chunk) > self.max_size: - # Chunk too large - split further - sub_chunks = self.strategies[self.fallback_strategy].chunk(chunk) - - # If still too large, use fixed-size chunking - final_sub_chunks = [] - for sub_chunk in sub_chunks: - if len(sub_chunk) > self.max_size: - final_sub_chunks.extend( - self.strategies['fixed'].chunk(sub_chunk) - ) - else: - final_sub_chunks.append(sub_chunk) - - refined_chunks.extend(final_sub_chunks) - else: - # Chunk too small - might merge with next or keep as is - refined_chunks.append(chunk) - - # Final size optimization - merge small adjacent chunks - optimized_chunks = self._merge_small_chunks(refined_chunks) - - return optimized_chunks - - def _merge_small_chunks(self, chunks: List[str]) -> List[str]: - """Merge small adjacent chunks to optimize sizes.""" - if not chunks: - return [] - - merged = [] - current = chunks[0] - - for i in range(1, len(chunks)): - next_chunk = chunks[i] - - # If both current and next are small, merge them - if (len(current) < self.target_size and - len(next_chunk) < self.target_size and - len(current + ' ' + next_chunk) <= self.max_size): - current = current + ' ' + next_chunk - else: - merged.append(current) - current = next_chunk - - # Add the last chunk - merged.append(current) - - return merged - - -# Factory function for creating chunking strategies -def create_chunking_strategy( - strategy_type: str, - config: Optional[Dict[str, Any]] = None -) -> ChunkingStrategy: - """ - Factory function to create chunking strategies. - - Args: - strategy_type: Type of strategy ('identity', 'regex', 'sentence', 'paragraph', - 'fixed', 'topic', 'hybrid') - config: Configuration dictionary for the strategy - - Returns: - Configured chunking strategy instance - """ - config = config or {} - - strategies = { - 'identity': IdentityChunking, - 'regex': RegexChunking, - 'sentence': SentenceChunking, - 'paragraph': ParagraphChunking, - 'fixed': FixedSizeChunking, - 'topic': TopicChunking, - 'hybrid': HybridChunking - } - - if strategy_type not in strategies: - raise ValueError(f"Unknown chunking strategy: {strategy_type}. Available: {list(strategies.keys())}") - - strategy_class = strategies[strategy_type] - return strategy_class(**config) - - -# Convenience functions -def chunk_text( - text: str, - strategy: str = 'paragraph', - config: Optional[Dict[str, Any]] = None -) -> List[str]: - """ - Convenience function to chunk text using specified strategy. - - Args: - text: Text to chunk - strategy: Chunking strategy to use - config: Strategy configuration - - Returns: - List of text chunks - """ - chunker = create_chunking_strategy(strategy, config) - return chunker.chunk(text) - - -def smart_chunk_for_llm( - text: str, - max_tokens: int = 4000, - overlap_tokens: int = 200, - strategy: str = 'hybrid' -) -> List[str]: - """ - Smart chunking optimized for LLM processing. - - Args: - text: Text to chunk - max_tokens: Maximum tokens per chunk (roughly 4 chars per token) - overlap_tokens: Tokens to overlap between chunks - strategy: Primary chunking strategy - - Returns: - List of text chunks optimized for LLM processing - """ - # Rough conversion: 1 token ≈ 4 characters - max_chars = max_tokens * 4 - overlap_chars = overlap_tokens * 4 - - if strategy == 'hybrid': - chunker = HybridChunking( - target_size=max_chars // 2, - max_size=max_chars, - min_size=100 - ) - else: - chunker = create_chunking_strategy(strategy, { - 'chunk_size': max_chars, - 'overlap': overlap_chars - }) - - return chunker.chunk(text) diff --git a/apps/backend/app/services/content_filters.py b/apps/backend/app/services/content_filters.py deleted file mode 100644 index 466d515..0000000 --- a/apps/backend/app/services/content_filters.py +++ /dev/null @@ -1,647 +0,0 @@ -""" -Advanced content filtering strategies inspired by crawl4ai. - -This module implements sophisticated content filtering for relevance and quality: -- BM25ContentFilter: Information retrieval-based filtering using BM25 algorithm -- PruningContentFilter: Removes irrelevant content based on thresholds -- LLMContentFilter: AI-powered content relevance filtering -- NoContentFilter: Pass-through filter for no filtering -""" - -import asyncio -import math -import re -import json -from abc import ABC, abstractmethod -from typing import List, Tuple, Dict, Optional, Set, Any -from collections import deque, Counter, defaultdict -from dataclasses import dataclass -import hashlib -from pathlib import Path - -import numpy as np -from bs4 import BeautifulSoup, Tag, NavigableString, Comment -from rank_bm25 import BM25Okapi -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.metrics.pairwise import cosine_similarity -import structlog - -from app.utils.text_processing import sanitize_text, clean_tokens - -logger = structlog.get_logger(__name__) - - -@dataclass -class FilterResult: - """Result of content filtering operation.""" - filtered_content: str - original_length: int - filtered_length: int - relevance_score: float - filter_metadata: Dict[str, Any] - - -class RelevantContentFilter(ABC): - """Abstract base class for content filtering strategies.""" - - def __init__( - self, - user_query: Optional[str] = None, - verbose: bool = False, - **kwargs - ): - """ - Initialize content filter. - - Args: - user_query: User query for relevance filtering (optional) - verbose: Enable verbose logging - **kwargs: Additional filter-specific parameters - """ - self.user_query = user_query - self.verbose = verbose - - # Tags to include in content filtering - self.included_tags = { - # Primary structure - "article", "main", "section", "div", - # List structures - "ul", "ol", "li", "dl", "dt", "dd", - # Text content - "p", "span", "blockquote", "pre", "code", - # Headers - "h1", "h2", "h3", "h4", "h5", "h6", - # Tables - "table", "thead", "tbody", "tr", "td", "th", - # Other semantic elements - "figure", "figcaption", "details", "summary", - # Text formatting - "em", "strong", "b", "i", "mark", "small", - # Rich content - "time", "address", "cite", "q" - } - - # Tags to exclude from content filtering - self.excluded_tags = { - "script", "style", "noscript", "nav", "footer", - "header", "aside", "form", "button", "input" - } - - @abstractmethod - async def filter(self, html_content: str, **kwargs) -> FilterResult: - """ - Filter HTML content based on relevance. - - Args: - html_content: Raw HTML content to filter - **kwargs: Additional filtering parameters - - Returns: - FilterResult with filtered content and metadata - """ - pass - - def _parse_html(self, html_content: str) -> BeautifulSoup: - """Parse HTML and clean unwanted elements.""" - soup = BeautifulSoup(html_content, 'lxml') - - # Remove excluded tags - for tag in self.excluded_tags: - for element in soup.find_all(tag): - element.decompose() - - return soup - - def _extract_text_blocks(self, soup: BeautifulSoup) -> List[Tuple[Tag, str, int]]: - """Extract text blocks with their elements and scores.""" - text_blocks = [] - - for element in soup.find_all(self.included_tags): - if self._should_include_element(element): - text = sanitize_text(element.get_text()) - if text and len(text.split()) >= 3: # Minimum word threshold - score = self._calculate_element_score(element) - text_blocks.append((element, text, score)) - - return text_blocks - - def _should_include_element(self, element: Tag) -> bool: - """Determine if element should be included in filtering.""" - # Skip if element is too nested in unimportant containers - parent_chain = [] - current = element.parent - while current and len(parent_chain) < 5: - if current.name: - parent_chain.append(current.name) - current = current.parent - - # Skip if too many div layers (likely layout) - if parent_chain.count('div') > 3: - return False - - # Include if has important semantic meaning - if element.name in ['article', 'main', 'section', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']: - return True - - # Include if has meaningful text content - text = element.get_text().strip() - return len(text) > 20 and len(text.split()) >= 5 - - def _calculate_element_score(self, element: Tag) -> int: - """Calculate relevance score for HTML element.""" - score = 0 - - # Tag-based scoring - tag_scores = { - 'h1': 10, 'h2': 8, 'h3': 6, 'h4': 4, 'h5': 2, 'h6': 1, - 'article': 8, 'main': 8, 'section': 6, - 'p': 3, 'div': 1, 'span': 1, - 'li': 2, 'td': 2, 'th': 3, - 'strong': 2, 'em': 1, 'b': 2, 'i': 1 - } - score += tag_scores.get(element.name, 0) - - # Class and ID based scoring - classes = element.get('class', []) - element_id = element.get('id', '') - - content_indicators = [ - 'content', 'article', 'main', 'body', 'text', - 'post', 'entry', 'story', 'news', 'blog' - ] - - for indicator in content_indicators: - if any(indicator in cls.lower() for cls in classes): - score += 3 - if indicator in element_id.lower(): - score += 3 - - return score - - -class NoContentFilter(RelevantContentFilter): - """Pass-through filter that doesn't modify content.""" - - async def filter(self, html_content: str, **kwargs) -> FilterResult: - """Return content without filtering.""" - soup = self._parse_html(html_content) - filtered_content = str(soup) - - return FilterResult( - filtered_content=filtered_content, - original_length=len(html_content), - filtered_length=len(filtered_content), - relevance_score=1.0, - filter_metadata={"filter_type": "none"} - ) - - -class PruningContentFilter(RelevantContentFilter): - """ - Remove irrelevant content based on configurable thresholds. - - This filter removes elements that don't meet minimum thresholds for: - - Word count - - Relevance score - - Content density - """ - - def __init__( - self, - threshold: float = 0.48, - threshold_type: str = "fixed", - min_word_threshold: int = 0, - **kwargs - ): - """ - Initialize pruning filter. - - Args: - threshold: Minimum relevance threshold (0.0 to 1.0) - threshold_type: Type of threshold ("fixed", "adaptive") - min_word_threshold: Minimum word count per element - """ - super().__init__(**kwargs) - self.threshold = threshold - self.threshold_type = threshold_type - self.min_word_threshold = min_word_threshold - - async def filter(self, html_content: str, **kwargs) -> FilterResult: - """Filter content by removing low-relevance elements.""" - soup = self._parse_html(html_content) - text_blocks = self._extract_text_blocks(soup) - - if not text_blocks: - return FilterResult( - filtered_content=html_content, - original_length=len(html_content), - filtered_length=len(html_content), - relevance_score=0.0, - filter_metadata={"filter_type": "pruning", "blocks_found": 0} - ) - - # Calculate adaptive threshold if needed - if self.threshold_type == "adaptive": - scores = [score for _, _, score in text_blocks] - mean_score = np.mean(scores) - std_score = np.std(scores) - adaptive_threshold = max(self.threshold, mean_score - 0.5 * std_score) - else: - adaptive_threshold = self.threshold - - # Filter elements based on thresholds - filtered_elements = [] - total_score = 0 - - for element, text, score in text_blocks: - word_count = len(text.split()) - - # Apply thresholds - normalized_score = score / 10.0 # Normalize to 0-1 range - - if (normalized_score >= adaptive_threshold and - word_count >= self.min_word_threshold): - filtered_elements.append(element) - total_score += score - - # Rebuild HTML with filtered elements - if filtered_elements: - # Create new soup with filtered content - new_soup = BeautifulSoup('
', 'lxml') - container = new_soup.find('div', class_='filtered-content') - - for element in filtered_elements: - # Clone element to avoid modifying original - cloned = BeautifulSoup(str(element), 'lxml').body.contents[0] - container.append(cloned) - - filtered_content = str(new_soup) - else: - # If nothing passes threshold, return original - filtered_content = html_content - - # Calculate relevance score - relevance_score = min(1.0, total_score / (len(text_blocks) * 10.0)) if text_blocks else 0.0 - - return FilterResult( - filtered_content=filtered_content, - original_length=len(html_content), - filtered_length=len(filtered_content), - relevance_score=relevance_score, - filter_metadata={ - "filter_type": "pruning", - "threshold": adaptive_threshold, - "blocks_original": len(text_blocks), - "blocks_filtered": len(filtered_elements), - "total_score": total_score - } - ) - - -class BM25ContentFilter(RelevantContentFilter): - """ - Information retrieval-based filtering using BM25 algorithm. - - This filter ranks content blocks by relevance to a user query using - the BM25 scoring algorithm, commonly used in search engines. - """ - - def __init__( - self, - user_query: str, - bm25_threshold: float = 1.0, - top_k: int = 10, - **kwargs - ): - """ - Initialize BM25 filter. - - Args: - user_query: Query to filter content against - bm25_threshold: Minimum BM25 score threshold - top_k: Maximum number of top-scoring blocks to include - """ - super().__init__(user_query=user_query, **kwargs) - self.bm25_threshold = bm25_threshold - self.top_k = top_k - - if not user_query: - raise ValueError("user_query is required for BM25ContentFilter") - - async def filter(self, html_content: str, **kwargs) -> FilterResult: - """Filter content using BM25 relevance scoring.""" - soup = self._parse_html(html_content) - text_blocks = self._extract_text_blocks(soup) - - if not text_blocks: - return FilterResult( - filtered_content=html_content, - original_length=len(html_content), - filtered_length=len(html_content), - relevance_score=0.0, - filter_metadata={"filter_type": "bm25", "blocks_found": 0} - ) - - # Prepare documents for BM25 - documents = [] - elements = [] - - for element, text, _ in text_blocks: - # Tokenize and clean text - tokens = clean_tokens(text.lower().split()) - if tokens: - documents.append(tokens) - elements.append(element) - - if not documents: - return FilterResult( - filtered_content=html_content, - original_length=len(html_content), - filtered_length=len(html_content), - relevance_score=0.0, - filter_metadata={"filter_type": "bm25", "documents": 0} - ) - - # Initialize BM25 - bm25 = BM25Okapi(documents) - - # Query tokenization - query_tokens = clean_tokens(self.user_query.lower().split()) - - # Get BM25 scores - bm25_scores = bm25.get_scores(query_tokens) - - # Combine elements with scores - scored_elements = list(zip(elements, bm25_scores)) - - # Filter by threshold and sort by score - filtered_scored = [ - (element, score) for element, score in scored_elements - if score >= self.bm25_threshold - ] - filtered_scored.sort(key=lambda x: x[1], reverse=True) - - # Take top-k elements - top_elements = filtered_scored[:self.top_k] - - if top_elements: - # Rebuild HTML with top elements in order of appearance - element_scores = {id(elem): score for elem, score in top_elements} - top_element_set = {id(elem) for elem, _ in top_elements} - - # Create filtered soup maintaining document order - new_soup = BeautifulSoup('
', 'lxml') - container = new_soup.find('div', class_='bm25-filtered-content') - - for element, _, _ in text_blocks: - if id(element) in top_element_set: - cloned = BeautifulSoup(str(element), 'lxml').body.contents[0] - container.append(cloned) - - filtered_content = str(new_soup) - - # Calculate average relevance score - avg_score = np.mean([score for _, score in top_elements]) - relevance_score = min(1.0, avg_score / 10.0) # Normalize BM25 score - else: - filtered_content = html_content - relevance_score = 0.0 - - return FilterResult( - filtered_content=filtered_content, - original_length=len(html_content), - filtered_length=len(filtered_content), - relevance_score=relevance_score, - filter_metadata={ - "filter_type": "bm25", - "query": self.user_query, - "threshold": self.bm25_threshold, - "blocks_original": len(text_blocks), - "blocks_filtered": len(top_elements), - "top_k": self.top_k, - "scores": [score for _, score in top_elements[:5]] # Top 5 scores - } - ) - - -class LLMContentFilter(RelevantContentFilter): - """ - AI-powered content relevance filtering using language models. - - This filter uses LLMs to determine content relevance based on: - - Natural language understanding - - Contextual relevance - - Query matching - """ - - def __init__( - self, - user_query: str, - llm_config: Optional[Dict[str, Any]] = None, - relevance_threshold: float = 0.7, - max_tokens: int = 4000, - **kwargs - ): - """ - Initialize LLM content filter. - - Args: - user_query: Query to filter content against - llm_config: Configuration for LLM provider - relevance_threshold: Minimum relevance threshold (0.0 to 1.0) - max_tokens: Maximum tokens to send to LLM - """ - super().__init__(user_query=user_query, **kwargs) - self.llm_config = llm_config or {} - self.relevance_threshold = relevance_threshold - self.max_tokens = max_tokens - - if not user_query: - raise ValueError("user_query is required for LLMContentFilter") - - async def filter(self, html_content: str, **kwargs) -> FilterResult: - """Filter content using LLM relevance assessment.""" - soup = self._parse_html(html_content) - text_blocks = self._extract_text_blocks(soup) - - if not text_blocks: - return FilterResult( - filtered_content=html_content, - original_length=len(html_content), - filtered_length=len(html_content), - relevance_score=0.0, - filter_metadata={"filter_type": "llm", "blocks_found": 0} - ) - - # Prepare content for LLM assessment - block_texts = [(element, text) for element, text, _ in text_blocks] - - # Truncate if too long for LLM context - total_chars = sum(len(text) for _, text in block_texts) - if total_chars > self.max_tokens * 4: # Rough char to token ratio - # Keep most important blocks first - sorted_blocks = sorted( - [(element, text, self._calculate_element_score(element)) - for element, text, _ in text_blocks], - key=lambda x: x[2], reverse=True - ) - - current_chars = 0 - truncated_blocks = [] - for element, text, score in sorted_blocks: - if current_chars + len(text) <= self.max_tokens * 4: - truncated_blocks.append((element, text)) - current_chars += len(text) - else: - break - block_texts = truncated_blocks - - # Assess relevance using LLM (mock implementation) - relevance_scores = await self._assess_relevance_with_llm(block_texts) - - # Filter blocks by relevance threshold - filtered_elements = [] - total_relevance = 0 - - for (element, text), relevance in zip(block_texts, relevance_scores): - if relevance >= self.relevance_threshold: - filtered_elements.append(element) - total_relevance += relevance - - # Rebuild HTML with relevant elements - if filtered_elements: - new_soup = BeautifulSoup('
', 'lxml') - container = new_soup.find('div', class_='llm-filtered-content') - - for element in filtered_elements: - cloned = BeautifulSoup(str(element), 'lxml').body.contents[0] - container.append(cloned) - - filtered_content = str(new_soup) - avg_relevance = total_relevance / len(filtered_elements) - else: - filtered_content = html_content - avg_relevance = 0.0 - - return FilterResult( - filtered_content=filtered_content, - original_length=len(html_content), - filtered_length=len(filtered_content), - relevance_score=avg_relevance, - filter_metadata={ - "filter_type": "llm", - "query": self.user_query, - "threshold": self.relevance_threshold, - "blocks_original": len(text_blocks), - "blocks_assessed": len(block_texts), - "blocks_filtered": len(filtered_elements), - "avg_relevance": avg_relevance - } - ) - - async def _assess_relevance_with_llm( - self, - block_texts: List[Tuple[Tag, str]] - ) -> List[float]: - """ - Assess content relevance using LLM. - - Note: This is a mock implementation. In production, integrate with - actual LLM providers like OpenAI, Anthropic, etc. - """ - # Mock implementation - simulate LLM assessment - await asyncio.sleep(0.2) # Simulate API call delay - - relevance_scores = [] - query_words = set(self.user_query.lower().split()) - - for element, text in block_texts: - # Simple keyword-based mock scoring - text_words = set(text.lower().split()) - overlap = len(query_words.intersection(text_words)) - max_overlap = len(query_words) - - # Mock relevance score based on keyword overlap - if max_overlap > 0: - base_score = overlap / max_overlap - # Add some randomness to simulate LLM assessment - import random - random.seed(hash(text[:50])) # Deterministic randomness - score = min(1.0, base_score + random.uniform(-0.2, 0.3)) - else: - score = 0.1 - - relevance_scores.append(max(0.0, score)) - - return relevance_scores - - -# Factory function for creating content filters -def create_content_filter( - filter_type: str, - config: Dict[str, Any] -) -> RelevantContentFilter: - """ - Factory function to create content filters. - - Args: - filter_type: Type of filter ("pruning", "bm25", "llm", "none") - config: Configuration dictionary for the filter - - Returns: - Configured content filter instance - """ - filters = { - "pruning": PruningContentFilter, - "bm25": BM25ContentFilter, - "llm": LLMContentFilter, - "none": NoContentFilter - } - - if filter_type not in filters: - raise ValueError(f"Unknown filter type: {filter_type}. Available: {list(filters.keys())}") - - filter_class = filters[filter_type] - return filter_class(**config) - - -# Convenience functions for common filtering patterns -async def filter_with_bm25( - html_content: str, - user_query: str, - bm25_threshold: float = 1.0, - top_k: int = 10 -) -> FilterResult: - """Filter content using BM25 algorithm.""" - filter_instance = BM25ContentFilter( - user_query=user_query, - bm25_threshold=bm25_threshold, - top_k=top_k - ) - return await filter_instance.filter(html_content) - - -async def filter_with_pruning( - html_content: str, - threshold: float = 0.48, - min_word_threshold: int = 0 -) -> FilterResult: - """Filter content using pruning strategy.""" - filter_instance = PruningContentFilter( - threshold=threshold, - min_word_threshold=min_word_threshold - ) - return await filter_instance.filter(html_content) - - -async def filter_with_llm( - html_content: str, - user_query: str, - relevance_threshold: float = 0.7 -) -> FilterResult: - """Filter content using LLM assessment.""" - filter_instance = LLMContentFilter( - user_query=user_query, - relevance_threshold=relevance_threshold - ) - return await filter_instance.filter(html_content) diff --git a/apps/backend/app/services/crawl_management.py b/apps/backend/app/services/crawl_management.py deleted file mode 100644 index 5309ce6..0000000 --- a/apps/backend/app/services/crawl_management.py +++ /dev/null @@ -1,829 +0,0 @@ -""" -Advanced Crawl Management service inspired by Firecrawl. - -Provides comprehensive crawl job lifecycle management: -- Job status monitoring and control -- Crawl job cancellation and error handling -- Real-time progress tracking -- Resource management and throttling -- Job queuing and prioritization -- Comprehensive error reporting -""" - -import asyncio -import time -import uuid -from typing import Dict, List, Optional, Any, Set -from dataclasses import dataclass, field -from datetime import datetime, timedelta -from enum import Enum -import structlog - -from app.config import get_settings -from app.models.responses import ScrapedContent -from app.services.enhanced_scraping import get_enhanced_scraping_service -from app.services.website_mapping import get_website_mapper, MapOptions, MapStrategy -from app.services.dispatcher import get_memory_adaptive_dispatcher - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class CrawlStatus(Enum): - """Status of crawl operations.""" - PENDING = "pending" - INITIALIZING = "initializing" - SCRAPING = "scraping" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - CANCELLED = "cancelled" - PAUSED = "paused" - - -class CrawlPriority(Enum): - """Priority levels for crawl jobs.""" - LOW = 1 - NORMAL = 5 - HIGH = 10 - URGENT = 20 - - -@dataclass -class CrawlError: - """Represents a crawl error.""" - url: str - error_type: str - message: str - timestamp: datetime = field(default_factory=datetime.utcnow) - retry_count: int = 0 - fatal: bool = False - - -@dataclass -class CrawlProgress: - """Progress tracking for crawl jobs.""" - total_urls: int = 0 - completed_urls: int = 0 - failed_urls: int = 0 - skipped_urls: int = 0 - current_url: Optional[str] = None - percentage: float = 0.0 - estimated_completion: Optional[datetime] = None - urls_per_minute: float = 0.0 - - -@dataclass -class CrawlOptions: - """Configuration options for crawl jobs.""" - # URL discovery - max_urls: int = 1000 - max_depth: int = 3 - include_subdomains: bool = True - allow_external_links: bool = False - ignore_sitemap: bool = False - include_paths: List[str] = field(default_factory=list) - exclude_paths: List[str] = field(default_factory=list) - - # Performance - max_concurrent_requests: int = 10 - delay_between_requests: float = 0.5 - timeout: int = 30 - - # Content filtering - include_tags: List[str] = field(default_factory=list) - exclude_tags: List[str] = field(default_factory=list) - only_main_content: bool = True - - # Retry logic - max_retries: int = 3 - retry_delay: float = 1.0 - - # Data retention - zero_data_retention: bool = False - webhook_url: Optional[str] = None - - -@dataclass -class CrawlJob: - """Represents a crawl job.""" - id: str - url: str - options: CrawlOptions - status: CrawlStatus = CrawlStatus.PENDING - priority: CrawlPriority = CrawlPriority.NORMAL - created_at: datetime = field(default_factory=datetime.utcnow) - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None - progress: CrawlProgress = field(default_factory=CrawlProgress) - errors: List[CrawlError] = field(default_factory=list) - results: List[ScrapedContent] = field(default_factory=list) - metadata: Dict[str, Any] = field(default_factory=dict) - - # Resource tracking - memory_usage_mb: float = 0.0 - cpu_usage_percent: float = 0.0 - - # Job control - cancellation_requested: bool = False - pause_requested: bool = False - - -@dataclass -class CrawlJobSummary: - """Summary information for a crawl job.""" - id: str - url: str - status: CrawlStatus - priority: CrawlPriority - created_at: datetime - progress_percentage: float - total_urls: int - completed_urls: int - error_count: int - estimated_completion: Optional[datetime] = None - - -class CrawlManager: - """ - Advanced crawl management service. - - Provides comprehensive crawl job lifecycle management including: - - Job creation, monitoring, and control - - Priority-based job queuing - - Resource management and throttling - - Error handling and retry logic - - Real-time progress tracking - """ - - def __init__(self): - """Initialize crawl manager.""" - self.jobs: Dict[str, CrawlJob] = {} - self.job_queue: List[str] = [] # Job IDs in priority order - self.active_jobs: Set[str] = set() - self.max_concurrent_jobs = getattr(settings, 'crawl_max_concurrent_jobs', 5) - - self.stats = { - "total_jobs": 0, - "completed_jobs": 0, - "failed_jobs": 0, - "cancelled_jobs": 0, - "total_urls_crawled": 0, - "total_errors": 0 - } - - # Start background worker - self._worker_task = None - asyncio.create_task(self._start_crawl_worker()) - - async def start_crawl( - self, - url: str, - options: Optional[CrawlOptions] = None, - priority: CrawlPriority = CrawlPriority.NORMAL, - metadata: Optional[Dict[str, Any]] = None - ) -> str: - """ - Start a new crawl job. - - Args: - url: Starting URL for the crawl - options: Crawl configuration options - priority: Job priority level - metadata: Additional metadata - - Returns: - Job ID for monitoring - """ - job_id = str(uuid.uuid4()) - options = options or CrawlOptions() - metadata = metadata or {} - - job = CrawlJob( - id=job_id, - url=url, - options=options, - priority=priority, - metadata=metadata - ) - - self.jobs[job_id] = job - self.stats["total_jobs"] += 1 - - # Add to priority queue - self._add_to_queue(job_id, priority) - - logger.info("crawl_job_created", - job_id=job_id, - url=url, - priority=priority.name, - max_urls=options.max_urls) - - return job_id - - async def get_crawl_status(self, job_id: str) -> Optional[Dict[str, Any]]: - """Get detailed status of a crawl job.""" - if job_id not in self.jobs: - return None - - job = self.jobs[job_id] - - return { - "id": job.id, - "url": job.url, - "status": job.status.value, - "priority": job.priority.name, - "created_at": job.created_at.isoformat(), - "started_at": job.started_at.isoformat() if job.started_at else None, - "completed_at": job.completed_at.isoformat() if job.completed_at else None, - "progress": { - "total_urls": job.progress.total_urls, - "completed_urls": job.progress.completed_urls, - "failed_urls": job.progress.failed_urls, - "skipped_urls": job.progress.skipped_urls, - "percentage": job.progress.percentage, - "current_url": job.progress.current_url, - "estimated_completion": job.progress.estimated_completion.isoformat() if job.progress.estimated_completion else None, - "urls_per_minute": job.progress.urls_per_minute - }, - "resource_usage": { - "memory_mb": job.memory_usage_mb, - "cpu_percent": job.cpu_usage_percent - }, - "errors": [ - { - "url": error.url, - "error_type": error.error_type, - "message": error.message, - "timestamp": error.timestamp.isoformat(), - "retry_count": error.retry_count, - "fatal": error.fatal - } - for error in job.errors[-10:] # Last 10 errors - ], - "total_errors": len(job.errors), - "results_count": len(job.results), - "can_cancel": job.status in [CrawlStatus.PENDING, CrawlStatus.SCRAPING, CrawlStatus.PAUSED], - "can_pause": job.status == CrawlStatus.SCRAPING, - "can_resume": job.status == CrawlStatus.PAUSED - } - - async def cancel_crawl(self, job_id: str) -> bool: - """Cancel a crawl job.""" - if job_id not in self.jobs: - return False - - job = self.jobs[job_id] - - if job.status in [CrawlStatus.COMPLETED, CrawlStatus.FAILED, CrawlStatus.CANCELLED]: - return False - - job.cancellation_requested = True - - # If job is active, mark it for immediate cancellation - if job_id in self.active_jobs: - job.status = CrawlStatus.CANCELLED - job.completed_at = datetime.utcnow() - self.active_jobs.remove(job_id) - self.stats["cancelled_jobs"] += 1 - else: - # Remove from queue if not started yet - if job_id in self.job_queue: - self.job_queue.remove(job_id) - job.status = CrawlStatus.CANCELLED - job.completed_at = datetime.utcnow() - self.stats["cancelled_jobs"] += 1 - - logger.info("crawl_job_cancelled", job_id=job_id) - - # Send webhook notification - if job.options.webhook_url: - await self._send_webhook_notification(job, "cancelled") - - return True - - async def pause_crawl(self, job_id: str) -> bool: - """Pause an active crawl job.""" - if job_id not in self.jobs or job_id not in self.active_jobs: - return False - - job = self.jobs[job_id] - - if job.status != CrawlStatus.SCRAPING: - return False - - job.pause_requested = True - job.status = CrawlStatus.PAUSED - - logger.info("crawl_job_paused", job_id=job_id) - return True - - async def resume_crawl(self, job_id: str) -> bool: - """Resume a paused crawl job.""" - if job_id not in self.jobs: - return False - - job = self.jobs[job_id] - - if job.status != CrawlStatus.PAUSED: - return False - - job.pause_requested = False - job.status = CrawlStatus.SCRAPING - - logger.info("crawl_job_resumed", job_id=job_id) - return True - - async def get_crawl_errors(self, job_id: str, limit: int = 100) -> List[Dict[str, Any]]: - """Get errors for a crawl job.""" - if job_id not in self.jobs: - return [] - - job = self.jobs[job_id] - errors = job.errors[-limit:] if limit > 0 else job.errors - - return [ - { - "url": error.url, - "error_type": error.error_type, - "message": error.message, - "timestamp": error.timestamp.isoformat(), - "retry_count": error.retry_count, - "fatal": error.fatal - } - for error in errors - ] - - async def get_active_crawls(self) -> List[CrawlJobSummary]: - """Get summary of all active crawl jobs.""" - active_summaries = [] - - for job_id in self.active_jobs: - if job_id in self.jobs: - job = self.jobs[job_id] - summary = CrawlJobSummary( - id=job.id, - url=job.url, - status=job.status, - priority=job.priority, - created_at=job.created_at, - progress_percentage=job.progress.percentage, - total_urls=job.progress.total_urls, - completed_urls=job.progress.completed_urls, - error_count=len(job.errors), - estimated_completion=job.progress.estimated_completion - ) - active_summaries.append(summary) - - # Sort by priority and creation time - active_summaries.sort(key=lambda x: (-x.priority.value, x.created_at)) - return active_summaries - - async def get_crawl_results(self, job_id: str, limit: int = 100) -> List[Dict[str, Any]]: - """Get results from a completed crawl job.""" - if job_id not in self.jobs: - return [] - - job = self.jobs[job_id] - results = job.results[-limit:] if limit > 0 else job.results - - return [result.dict() for result in results] - - def _add_to_queue(self, job_id: str, priority: CrawlPriority): - """Add job to priority queue.""" - # Find insertion point based on priority - insert_pos = 0 - for i, existing_job_id in enumerate(self.job_queue): - if existing_job_id in self.jobs: - existing_priority = self.jobs[existing_job_id].priority - if priority.value > existing_priority.value: - break - insert_pos = i + 1 - - self.job_queue.insert(insert_pos, job_id) - - async def _start_crawl_worker(self): - """Start background worker for processing crawl jobs.""" - self._worker_task = asyncio.create_task(self._crawl_worker()) - - async def _crawl_worker(self): - """Background worker that processes crawl jobs.""" - while True: - try: - await self._process_job_queue() - await asyncio.sleep(5) # Check queue every 5 seconds - - except Exception as e: - logger.error("crawl_worker_error", error=str(e)) - await asyncio.sleep(30) # Wait 30 seconds before retrying - - async def _process_job_queue(self): - """Process jobs from the queue.""" - # Check if we can start new jobs - if len(self.active_jobs) >= self.max_concurrent_jobs: - return - - # Clean up completed jobs from active set - completed_jobs = [] - for job_id in list(self.active_jobs): - if job_id in self.jobs: - job = self.jobs[job_id] - if job.status in [CrawlStatus.COMPLETED, CrawlStatus.FAILED, CrawlStatus.CANCELLED]: - completed_jobs.append(job_id) - - for job_id in completed_jobs: - self.active_jobs.remove(job_id) - - # Start new jobs from queue - while len(self.active_jobs) < self.max_concurrent_jobs and self.job_queue: - job_id = self.job_queue.pop(0) - - if job_id in self.jobs: - job = self.jobs[job_id] - - # Skip cancelled jobs - if job.cancellation_requested: - job.status = CrawlStatus.CANCELLED - job.completed_at = datetime.utcnow() - continue - - # Start the job - self.active_jobs.add(job_id) - asyncio.create_task(self._execute_crawl_job(job)) - - async def _execute_crawl_job(self, job: CrawlJob): - """Execute a single crawl job.""" - try: - job.status = CrawlStatus.INITIALIZING - job.started_at = datetime.utcnow() - - logger.info("crawl_job_started", - job_id=job.id, - url=job.url) - - # Step 1: URL Discovery - await self._discover_urls_for_job(job) - - if job.cancellation_requested: - job.status = CrawlStatus.CANCELLED - job.completed_at = datetime.utcnow() - return - - # Step 2: Scraping - job.status = CrawlStatus.SCRAPING - await self._scrape_urls_for_job(job) - - if job.cancellation_requested: - job.status = CrawlStatus.CANCELLED - job.completed_at = datetime.utcnow() - return - - # Step 3: Post-processing - job.status = CrawlStatus.PROCESSING - await self._post_process_job(job) - - # Complete job - job.status = CrawlStatus.COMPLETED - job.completed_at = datetime.utcnow() - job.progress.percentage = 100.0 - - self.stats["completed_jobs"] += 1 - self.stats["total_urls_crawled"] += job.progress.completed_urls - - logger.info("crawl_job_completed", - job_id=job.id, - urls_crawled=job.progress.completed_urls, - processing_time_ms=int((job.completed_at - job.started_at).total_seconds() * 1000)) - - # Send webhook notification - if job.options.webhook_url: - await self._send_webhook_notification(job, "completed") - - except Exception as e: - job.status = CrawlStatus.FAILED - job.completed_at = datetime.utcnow() - job.errors.append(CrawlError( - url=job.url, - error_type="job_execution_error", - message=str(e), - fatal=True - )) - - self.stats["failed_jobs"] += 1 - self.stats["total_errors"] += 1 - - logger.error("crawl_job_failed", - job_id=job.id, - error=str(e)) - - # Send webhook notification - if job.options.webhook_url: - await self._send_webhook_notification(job, "failed") - - async def _discover_urls_for_job(self, job: CrawlJob): - """Discover URLs for crawling.""" - try: - mapper = await get_website_mapper() - - map_options = MapOptions( - strategy=MapStrategy.COMBINED, - limit=job.options.max_urls, - include_subdomains=job.options.include_subdomains, - allow_external_links=job.options.allow_external_links, - ignore_sitemap=job.options.ignore_sitemap, - max_depth=job.options.max_depth - ) - - mapping_result = await mapper.map_website(job.url, map_options) - - if mapping_result.success: - discovered_urls = [du.url for du in mapping_result.discovered_urls] - - # Apply include/exclude path filters - filtered_urls = self._apply_path_filters(discovered_urls, job.options) - - job.progress.total_urls = len(filtered_urls) - job.metadata["discovered_urls"] = filtered_urls - - logger.info("urls_discovered_for_job", - job_id=job.id, - total_urls=len(filtered_urls)) - else: - raise Exception(f"URL discovery failed: {mapping_result.error}") - - except Exception as e: - job.errors.append(CrawlError( - url=job.url, - error_type="url_discovery_error", - message=str(e) - )) - raise e - - def _apply_path_filters(self, urls: List[str], options: CrawlOptions) -> List[str]: - """Apply include/exclude path filters.""" - filtered_urls = urls - - # Apply include patterns - if options.include_paths: - filtered_urls = [ - url for url in filtered_urls - if any(pattern in url for pattern in options.include_paths) - ] - - # Apply exclude patterns - if options.exclude_paths: - filtered_urls = [ - url for url in filtered_urls - if not any(pattern in url for pattern in options.exclude_paths) - ] - - return filtered_urls - - async def _scrape_urls_for_job(self, job: CrawlJob): - """Scrape URLs for the job.""" - urls = job.metadata.get("discovered_urls", []) - if not urls: - return - - scraping_service = await get_enhanced_scraping_service() - dispatcher = await get_memory_adaptive_dispatcher() - - # Process URLs in batches - batch_size = min(job.options.max_concurrent_requests, 20) - start_time = time.time() - - for i in range(0, len(urls), batch_size): - # Check for cancellation/pause - if job.cancellation_requested: - break - - while job.pause_requested: - await asyncio.sleep(1) - if job.cancellation_requested: - break - - batch_urls = urls[i:i + batch_size] - job.progress.current_url = batch_urls[0] if batch_urls else None - - try: - # Use dispatcher for rate limiting - batch_results = await dispatcher.dispatch_async_batch( - [scraping_service.scrape_urls_enhanced([url]) for url in batch_urls] - ) - - # Process results - for j, result_list in enumerate(batch_results): - url = batch_urls[j] - - if result_list and result_list[0].extraction_success: - job.results.append(result_list[0]) - job.progress.completed_urls += 1 - else: - job.progress.failed_urls += 1 - job.errors.append(CrawlError( - url=url, - error_type="scraping_failed", - message="Failed to scrape URL" - )) - self.stats["total_errors"] += 1 - - # Update progress - total_processed = job.progress.completed_urls + job.progress.failed_urls - job.progress.percentage = min(90.0, (total_processed / job.progress.total_urls) * 90.0) - - # Calculate URLs per minute - elapsed_time = time.time() - start_time - if elapsed_time > 0: - job.progress.urls_per_minute = (total_processed / elapsed_time) * 60 - - # Estimate completion time - if job.progress.urls_per_minute > 0: - remaining_urls = job.progress.total_urls - total_processed - remaining_minutes = remaining_urls / job.progress.urls_per_minute - job.progress.estimated_completion = datetime.utcnow() + timedelta(minutes=remaining_minutes) - - # Apply delay between batches - if job.options.delay_between_requests > 0: - await asyncio.sleep(job.options.delay_between_requests) - - except Exception as e: - logger.error("batch_scraping_failed", - job_id=job.id, - batch_urls=batch_urls, - error=str(e)) - - # Mark all URLs in batch as failed - for url in batch_urls: - job.progress.failed_urls += 1 - job.errors.append(CrawlError( - url=url, - error_type="batch_scraping_error", - message=str(e) - )) - - logger.info("scraping_completed_for_job", - job_id=job.id, - completed_urls=job.progress.completed_urls, - failed_urls=job.progress.failed_urls) - - async def _post_process_job(self, job: CrawlJob): - """Post-process job results.""" - # Register for zero data retention if enabled - if job.options.zero_data_retention: - from app.services.zero_retention import get_zero_retention_manager, DataType - - try: - retention_manager = await get_zero_retention_manager() - - # Calculate total size - total_size = sum( - len(result.text or "") + len(result.html or "") - for result in job.results - ) - - await retention_manager.register_data( - data_id=f"crawl_job_{job.id}", - data_type=DataType.CRAWL_DATA, - size_bytes=total_size, - tags=["zero_retention", "crawl_job"], - secure_delete=True - ) - - logger.info("job_registered_for_zero_retention", - job_id=job.id, - size_bytes=total_size) - - except Exception as e: - logger.warning("zero_retention_registration_failed", - job_id=job.id, - error=str(e)) - - async def _send_webhook_notification(self, job: CrawlJob, event: str): - """Send webhook notification about job status.""" - try: - import httpx - - webhook_data = { - "job_id": job.id, - "url": job.url, - "event": event, - "status": job.status.value, - "progress": { - "total_urls": job.progress.total_urls, - "completed_urls": job.progress.completed_urls, - "failed_urls": job.progress.failed_urls, - "percentage": job.progress.percentage - }, - "timestamp": datetime.utcnow().isoformat(), - "results_count": len(job.results), - "errors_count": len(job.errors) - } - - async with httpx.AsyncClient() as client: - response = await client.post( - job.options.webhook_url, - json=webhook_data, - timeout=10 - ) - - if response.status_code == 200: - logger.info("webhook_notification_sent", - job_id=job.id, - event=event, - webhook_url=job.options.webhook_url) - else: - logger.warning("webhook_notification_failed", - job_id=job.id, - event=event, - status_code=response.status_code) - - except Exception as e: - logger.error("webhook_notification_error", - job_id=job.id, - event=event, - error=str(e)) - - async def get_crawl_stats(self) -> Dict[str, Any]: - """Get crawl management statistics.""" - return { - "crawl_stats": self.stats, - "active_jobs": len(self.active_jobs), - "queued_jobs": len(self.job_queue), - "total_jobs_managed": len(self.jobs), - "max_concurrent_jobs": self.max_concurrent_jobs, - "success_rate": ( - self.stats["completed_jobs"] / - max(1, self.stats["total_jobs"]) - ) if self.stats["total_jobs"] > 0 else 0, - "avg_urls_per_job": ( - self.stats["total_urls_crawled"] / - max(1, self.stats["completed_jobs"]) - ) if self.stats["completed_jobs"] > 0 else 0, - "worker_running": self._worker_task is not None and not self._worker_task.done() - } - - async def cleanup(self): - """Cleanup resources.""" - if self._worker_task: - self._worker_task.cancel() - try: - await self._worker_task - except asyncio.CancelledError: - pass - - -# Singleton service -_crawl_manager: Optional[CrawlManager] = None - - -async def get_crawl_manager() -> CrawlManager: - """Get or create crawl manager service instance.""" - global _crawl_manager - - if _crawl_manager is None: - _crawl_manager = CrawlManager() - - return _crawl_manager - - -# Convenience functions -async def start_website_crawl( - url: str, - max_urls: int = 100, - max_depth: int = 2, - webhook_url: Optional[str] = None, - zero_data_retention: bool = False -) -> str: - """ - Start a website crawl with common options. - - Args: - url: Starting URL - max_urls: Maximum URLs to crawl - max_depth: Maximum crawl depth - webhook_url: Webhook for notifications - zero_data_retention: Enable 24-hour data deletion - - Returns: - Job ID for monitoring - """ - manager = await get_crawl_manager() - - options = CrawlOptions( - max_urls=max_urls, - max_depth=max_depth, - webhook_url=webhook_url, - zero_data_retention=zero_data_retention - ) - - return await manager.start_crawl(url, options) - - -async def get_crawl_progress(job_id: str) -> Optional[Dict[str, Any]]: - """Get crawl job progress.""" - manager = await get_crawl_manager() - return await manager.get_crawl_status(job_id) - - - - diff --git a/apps/backend/app/services/crawler_monitor.py b/apps/backend/app/services/crawler_monitor.py deleted file mode 100644 index c06bb01..0000000 --- a/apps/backend/app/services/crawler_monitor.py +++ /dev/null @@ -1,568 +0,0 @@ -""" -Advanced crawler monitoring system with real-time status tracking and metrics. - -This module provides comprehensive monitoring capabilities: -- Real-time crawling status and progress tracking -- Performance metrics collection -- Resource usage monitoring -- Terminal UI for live monitoring -- Event logging and statistics -- Configurable alerts and thresholds -""" - -import time -import threading -import asyncio -from typing import Dict, List, Optional, Any, Callable -from dataclasses import dataclass, field -from enum import Enum -from datetime import datetime, timedelta -from collections import deque, defaultdict - -import psutil -import structlog - -logger = structlog.get_logger(__name__) - - -class CrawlStatus(str, Enum): - """Crawling status enumeration.""" - IDLE = "idle" - STARTING = "starting" - RUNNING = "running" - PAUSED = "paused" - STOPPING = "stopping" - COMPLETED = "completed" - ERROR = "error" - - -@dataclass -class TaskMetrics: - """Metrics for individual crawler tasks.""" - task_id: str - url: str - status: CrawlStatus - start_time: float - end_time: Optional[float] = None - response_time: Optional[float] = None - status_code: Optional[int] = None - content_length: int = 0 - error: Optional[str] = None - - @property - def duration(self) -> float: - """Get task duration in seconds.""" - if self.end_time: - return self.end_time - self.start_time - return time.time() - self.start_time - - @property - def is_completed(self) -> bool: - """Check if task is completed.""" - return self.status in [CrawlStatus.COMPLETED, CrawlStatus.ERROR] - - -@dataclass -class SystemMetrics: - """System resource metrics.""" - timestamp: float - cpu_percent: float - memory_percent: float - memory_used_mb: float - disk_io_read_mb: float - disk_io_write_mb: float - network_io_sent_mb: float - network_io_recv_mb: float - active_connections: int = 0 - - @classmethod - def current(cls) -> "SystemMetrics": - """Get current system metrics.""" - cpu_percent = psutil.cpu_percent(interval=0.1) - memory = psutil.virtual_memory() - disk_io = psutil.disk_io_counters() - network_io = psutil.net_io_counters() - - return cls( - timestamp=time.time(), - cpu_percent=cpu_percent, - memory_percent=memory.percent, - memory_used_mb=memory.used / 1024 / 1024, - disk_io_read_mb=disk_io.read_bytes / 1024 / 1024 if disk_io else 0, - disk_io_write_mb=disk_io.write_bytes / 1024 / 1024 if disk_io else 0, - network_io_sent_mb=network_io.bytes_sent / 1024 / 1024 if network_io else 0, - network_io_recv_mb=network_io.bytes_recv / 1024 / 1024 if network_io else 0 - ) - - -@dataclass -class CrawlerStats: - """Comprehensive crawler statistics.""" - start_time: float = field(default_factory=time.time) - end_time: Optional[float] = None - - # Task counters - total_tasks: int = 0 - completed_tasks: int = 0 - failed_tasks: int = 0 - active_tasks: int = 0 - - # Performance metrics - avg_response_time: float = 0.0 - min_response_time: float = float('inf') - max_response_time: float = 0.0 - total_bytes_downloaded: int = 0 - - # Status code distribution - status_codes: Dict[int, int] = field(default_factory=lambda: defaultdict(int)) - - # Error tracking - errors: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) - - # Rate metrics - current_rate: float = 0.0 # requests per second - peak_rate: float = 0.0 - - @property - def success_rate(self) -> float: - """Calculate success rate.""" - if self.total_tasks == 0: - return 0.0 - return (self.completed_tasks / self.total_tasks) * 100 - - @property - def duration(self) -> float: - """Get total crawling duration.""" - end = self.end_time or time.time() - return end - self.start_time - - @property - def throughput(self) -> float: - """Calculate overall throughput (requests per second).""" - duration = self.duration - if duration > 0: - return self.total_tasks / duration - return 0.0 - - -class CrawlerMonitor: - """ - Advanced crawler monitoring system. - - Provides real-time monitoring, metrics collection, and performance tracking - for web crawling operations. - """ - - def __init__(self, - update_interval: float = 1.0, - max_history_size: int = 1000, - enable_terminal_ui: bool = False): - """ - Initialize crawler monitor. - - Args: - update_interval: Update interval for metrics collection - max_history_size: Maximum number of metrics to keep in history - enable_terminal_ui: Whether to enable terminal UI - """ - self.update_interval = update_interval - self.max_history_size = max_history_size - self.enable_terminal_ui = enable_terminal_ui - - # Monitoring state - self.is_running = False - self._monitor_thread: Optional[threading.Thread] = None - self._stop_event = threading.Event() - - # Statistics - self.stats = CrawlerStats() - self.active_tasks: Dict[str, TaskMetrics] = {} - self.completed_tasks: deque = deque(maxlen=max_history_size) - - # System metrics history - self.system_metrics_history: deque = deque(maxlen=max_history_size) - - # Rate tracking - self._rate_window = deque(maxlen=60) # 1 minute window - - # Event callbacks - self._event_callbacks: Dict[str, List[Callable]] = defaultdict(list) - - # Terminal UI - self._terminal_ui = None - if enable_terminal_ui: - try: - self._terminal_ui = TerminalUI() - except ImportError: - logger.warning("Terminal UI dependencies not available") - - def start(self): - """Start the monitoring system.""" - if self.is_running: - return - - self.is_running = True - self._stop_event.clear() - - # Start monitoring thread - self._monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True) - self._monitor_thread.start() - - # Start terminal UI if enabled - if self._terminal_ui: - self._terminal_ui.start(self) - - logger.info("Crawler monitor started") - self._trigger_event('monitor_started', {}) - - def stop(self): - """Stop the monitoring system.""" - if not self.is_running: - return - - self.is_running = False - self._stop_event.set() - - # Stop terminal UI - if self._terminal_ui: - self._terminal_ui.stop() - - # Wait for monitor thread - if self._monitor_thread and self._monitor_thread.is_alive(): - self._monitor_thread.join(timeout=5.0) - - # Mark stats end time - self.stats.end_time = time.time() - - logger.info("Crawler monitor stopped") - self._trigger_event('monitor_stopped', {'stats': self.stats}) - - def _monitor_loop(self): - """Main monitoring loop.""" - while not self._stop_event.is_set(): - try: - # Update system metrics - system_metrics = SystemMetrics.current() - system_metrics.active_connections = len(self.active_tasks) - self.system_metrics_history.append(system_metrics) - - # Update rate metrics - self._update_rate_metrics() - - # Check for alerts - self._check_alerts(system_metrics) - - # Sleep until next update - self._stop_event.wait(self.update_interval) - - except Exception as e: - logger.error(f"Error in monitor loop: {str(e)}") - time.sleep(self.update_interval) - - def _update_rate_metrics(self): - """Update rate-based metrics.""" - current_time = time.time() - current_active = len(self.active_tasks) - - # Add current measurement to rate window - self._rate_window.append((current_time, current_active)) - - # Calculate current rate (requests per second over last 10 seconds) - ten_seconds_ago = current_time - 10.0 - recent_measurements = [(t, count) for t, count in self._rate_window if t > ten_seconds_ago] - - if len(recent_measurements) >= 2: - # Calculate rate based on change in active tasks - start_time, start_count = recent_measurements[0] - end_time, end_count = recent_measurements[-1] - - time_diff = end_time - start_time - if time_diff > 0: - # This is a simplified rate calculation - completed_in_window = max(0, self.stats.completed_tasks - getattr(self, '_last_completed_count', 0)) - self.stats.current_rate = completed_in_window / time_diff - - # Update peak rate - if self.stats.current_rate > self.stats.peak_rate: - self.stats.peak_rate = self.stats.current_rate - - # Store count for next calculation - self._last_completed_count = self.stats.completed_tasks - - def _check_alerts(self, system_metrics: SystemMetrics): - """Check for alert conditions.""" - alerts = [] - - # High CPU usage - if system_metrics.cpu_percent > 90: - alerts.append({ - 'type': 'high_cpu', - 'message': f'High CPU usage: {system_metrics.cpu_percent:.1f}%', - 'severity': 'warning' - }) - - # High memory usage - if system_metrics.memory_percent > 85: - alerts.append({ - 'type': 'high_memory', - 'message': f'High memory usage: {system_metrics.memory_percent:.1f}%', - 'severity': 'warning' - }) - - # Low success rate - if self.stats.total_tasks > 10 and self.stats.success_rate < 50: - alerts.append({ - 'type': 'low_success_rate', - 'message': f'Low success rate: {self.stats.success_rate:.1f}%', - 'severity': 'error' - }) - - # Trigger alert events - for alert in alerts: - self._trigger_event('alert', alert) - - def task_started(self, task_id: str, url: str): - """Record task start.""" - task_metrics = TaskMetrics( - task_id=task_id, - url=url, - status=CrawlStatus.STARTING, - start_time=time.time() - ) - - self.active_tasks[task_id] = task_metrics - self.stats.total_tasks += 1 - self.stats.active_tasks = len(self.active_tasks) - - self._trigger_event('task_started', {'task_id': task_id, 'url': url}) - - def task_completed(self, - task_id: str, - status_code: Optional[int] = None, - content_length: int = 0, - response_time: Optional[float] = None): - """Record task completion.""" - task_metrics = self.active_tasks.get(task_id) - if not task_metrics: - return - - # Update task metrics - task_metrics.end_time = time.time() - task_metrics.status = CrawlStatus.COMPLETED - task_metrics.status_code = status_code - task_metrics.content_length = content_length - task_metrics.response_time = response_time or task_metrics.duration - - # Update statistics - self.stats.completed_tasks += 1 - self.stats.active_tasks = len(self.active_tasks) - 1 - self.stats.total_bytes_downloaded += content_length - - if status_code: - self.stats.status_codes[status_code] += 1 - - if task_metrics.response_time: - # Update response time statistics - rt = task_metrics.response_time - self.stats.min_response_time = min(self.stats.min_response_time, rt) - self.stats.max_response_time = max(self.stats.max_response_time, rt) - - # Update average response time - total_completed = self.stats.completed_tasks - if total_completed > 1: - self.stats.avg_response_time = ( - (self.stats.avg_response_time * (total_completed - 1) + rt) / total_completed - ) - else: - self.stats.avg_response_time = rt - - # Move to completed tasks - self.completed_tasks.append(task_metrics) - del self.active_tasks[task_id] - - self._trigger_event('task_completed', { - 'task_id': task_id, - 'status_code': status_code, - 'response_time': task_metrics.response_time - }) - - def task_failed(self, task_id: str, error: str): - """Record task failure.""" - task_metrics = self.active_tasks.get(task_id) - if not task_metrics: - return - - # Update task metrics - task_metrics.end_time = time.time() - task_metrics.status = CrawlStatus.ERROR - task_metrics.error = error - - # Update statistics - self.stats.failed_tasks += 1 - self.stats.active_tasks = len(self.active_tasks) - 1 - self.stats.errors[error] += 1 - - # Move to completed tasks - self.completed_tasks.append(task_metrics) - del self.active_tasks[task_id] - - self._trigger_event('task_failed', {'task_id': task_id, 'error': error}) - - def on_event(self, event_type: str, callback: Callable): - """Register event callback.""" - self._event_callbacks[event_type].append(callback) - - def _trigger_event(self, event_type: str, data: Dict[str, Any]): - """Trigger event callbacks.""" - for callback in self._event_callbacks[event_type]: - try: - callback(data) - except Exception as e: - logger.error(f"Error in event callback for {event_type}: {str(e)}") - - def get_current_metrics(self) -> Dict[str, Any]: - """Get current monitoring metrics.""" - current_system = self.system_metrics_history[-1] if self.system_metrics_history else None - - return { - 'timestamp': time.time(), - 'status': CrawlStatus.RUNNING if self.is_running else CrawlStatus.IDLE, - 'stats': { - 'total_tasks': self.stats.total_tasks, - 'completed_tasks': self.stats.completed_tasks, - 'failed_tasks': self.stats.failed_tasks, - 'active_tasks': self.stats.active_tasks, - 'success_rate': self.stats.success_rate, - 'avg_response_time': self.stats.avg_response_time, - 'current_rate': self.stats.current_rate, - 'peak_rate': self.stats.peak_rate, - 'total_bytes': self.stats.total_bytes_downloaded, - 'duration': self.stats.duration - }, - 'system': { - 'cpu_percent': current_system.cpu_percent if current_system else 0, - 'memory_percent': current_system.memory_percent if current_system else 0, - 'memory_used_mb': current_system.memory_used_mb if current_system else 0, - 'active_connections': current_system.active_connections if current_system else 0 - }, - 'top_errors': dict(list(self.stats.errors.items())[:5]), - 'status_codes': dict(self.stats.status_codes) - } - - def get_performance_report(self) -> Dict[str, Any]: - """Generate comprehensive performance report.""" - metrics = self.get_current_metrics() - - # Calculate additional performance indicators - report = { - 'summary': { - 'total_duration': self.stats.duration, - 'total_tasks': self.stats.total_tasks, - 'success_rate': self.stats.success_rate, - 'throughput': self.stats.throughput, - 'total_data': self.stats.total_bytes_downloaded - }, - 'performance': { - 'avg_response_time': self.stats.avg_response_time, - 'min_response_time': self.stats.min_response_time if self.stats.min_response_time != float('inf') else 0, - 'max_response_time': self.stats.max_response_time, - 'current_rate': self.stats.current_rate, - 'peak_rate': self.stats.peak_rate - }, - 'system_usage': metrics['system'], - 'errors': dict(self.stats.errors), - 'status_codes': dict(self.stats.status_codes), - 'recommendations': self._generate_recommendations() - } - - return report - - def _generate_recommendations(self) -> List[str]: - """Generate performance recommendations.""" - recommendations = [] - - # Success rate recommendations - if self.stats.total_tasks > 10: - if self.stats.success_rate < 70: - recommendations.append("Low success rate detected. Check network connectivity and target server stability.") - elif self.stats.success_rate < 90: - recommendations.append("Consider implementing retry logic for failed requests.") - - # Response time recommendations - if self.stats.avg_response_time > 5.0: - recommendations.append("High average response time. Consider reducing concurrent requests or increasing timeout.") - - # System resource recommendations - current_system = self.system_metrics_history[-1] if self.system_metrics_history else None - if current_system: - if current_system.cpu_percent > 80: - recommendations.append("High CPU usage. Consider reducing concurrency or optimizing processing logic.") - - if current_system.memory_percent > 80: - recommendations.append("High memory usage. Consider implementing content streaming or reducing cache sizes.") - - # Error pattern recommendations - common_errors = sorted(self.stats.errors.items(), key=lambda x: x[1], reverse=True)[:3] - for error, count in common_errors: - if count > self.stats.total_tasks * 0.1: # More than 10% of tasks - recommendations.append(f"Frequent error '{error}' ({count} occurrences). Investigate root cause.") - - return recommendations - - -# Simplified Terminal UI stub (full implementation would require rich/curses) -class TerminalUI: - """Simplified terminal UI for monitoring (stub implementation).""" - - def __init__(self): - self.is_running = False - self.monitor = None - - def start(self, monitor: CrawlerMonitor): - """Start terminal UI.""" - self.monitor = monitor - self.is_running = True - logger.info("Terminal UI started (simplified mode)") - - def stop(self): - """Stop terminal UI.""" - self.is_running = False - logger.info("Terminal UI stopped") - - -# Factory functions -def create_crawler_monitor( - update_interval: float = 1.0, - max_history_size: int = 1000, - enable_terminal_ui: bool = False -) -> CrawlerMonitor: - """Create a crawler monitor instance.""" - return CrawlerMonitor( - update_interval=update_interval, - max_history_size=max_history_size, - enable_terminal_ui=enable_terminal_ui - ) - - -# Singleton instance for global monitoring -_global_monitor: Optional[CrawlerMonitor] = None - - -def get_global_monitor() -> CrawlerMonitor: - """Get global crawler monitor instance.""" - global _global_monitor - if _global_monitor is None: - _global_monitor = create_crawler_monitor() - return _global_monitor - - -def start_global_monitoring(): - """Start global monitoring.""" - monitor = get_global_monitor() - monitor.start() - - -def stop_global_monitoring(): - """Stop global monitoring.""" - monitor = get_global_monitor() - monitor.stop() diff --git a/apps/backend/app/services/database.py b/apps/backend/app/services/database.py deleted file mode 100644 index 75b6547..0000000 --- a/apps/backend/app/services/database.py +++ /dev/null @@ -1,464 +0,0 @@ -""" -Database service for persistent storage operations. -""" -from typing import Optional, List, Dict, Any -from datetime import datetime, timedelta -from contextlib import asynccontextmanager -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker -from sqlalchemy.orm import selectinload -from sqlalchemy import select, func, and_, or_, desc -from sqlalchemy.exc import IntegrityError -import uuid - -from app.config import get_settings -from app.models.database import ( - Base, APIKey, SearchRequest, SearchResult as DBSearchResult, - ScrapingJob, CacheEntry, ErrorLog -) -from app.models.responses import UnQuestResponse -import structlog -import ssl -from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class DatabaseService: - """Service for database operations.""" - - def __init__(self): - # Convert sync PostgreSQL URL to async and handle sslmode for asyncpg (Neon) - original_url = str(settings.database_url) - if original_url.startswith("postgresql://"): - async_url = original_url.replace("postgresql://", "postgresql+asyncpg://", 1) - elif original_url.startswith("postgres://"): - async_url = original_url.replace("postgres://", "postgresql+asyncpg://", 1) - else: - async_url = original_url - - # Strip unsupported sslmode from asyncpg URL and map to connect_args['ssl'] - connect_args: dict[str, object] = {} - try: - parts = urlsplit(async_url) - query_items = parse_qsl(parts.query, keep_blank_values=True) - sslmode_value = None - filtered_items = [] - # Remove parameters not understood by asyncpg - unsupported_params = { - "sslmode", - "sslrootcert", - "sslcert", - "sslkey", - "options", - "channel_binding", - } - for k, v in query_items: - key_lower = k.lower() - if key_lower == "sslmode": - sslmode_value = v.lower() if isinstance(v, str) else str(v) - # drop from URL - continue - if key_lower in unsupported_params: - # drop from URL; handled via connect_args where applicable - continue - else: - filtered_items.append((k, v)) - - # asyncpg doesn't accept 'sslmode'; use 'ssl' connect arg instead - if sslmode_value and sslmode_value != "disable": - # For Neon, a default SSL context or True works; True lets asyncpg create a context - connect_args["ssl"] = True - - cleaned_query = urlencode(filtered_items, doseq=True) - async_url = urlunsplit((parts.scheme, parts.netloc, parts.path, cleaned_query, parts.fragment)) - except Exception: - # If anything goes wrong, fall back to the async_url as-is - pass - - self.engine = create_async_engine( - async_url, - pool_size=settings.database_pool_size, - max_overflow=settings.database_max_overflow, - pool_timeout=settings.database_pool_timeout, - echo=settings.database_echo, - future=True, - connect_args=connect_args, - ) - - self.async_session = async_sessionmaker( - self.engine, - class_=AsyncSession, - expire_on_commit=False - ) - - async def initialize(self): - """Initialize database connection. - - Avoid creating schema at startup to prevent greenlet requirement and - rely on Alembic migrations for schema management. Just validate the - connection with a lightweight query. - """ - from sqlalchemy import text - async with self.engine.connect() as conn: - await conn.execute(text("SELECT 1")) - logger.info("database_initialized") - - async def close(self): - """Close database connections.""" - await self.engine.dispose() - logger.info("database_closed") - - @asynccontextmanager - async def get_session(self): - """Get database session context manager.""" - async with self.async_session() as session: - try: - yield session - await session.commit() - except Exception: - await session.rollback() - raise - finally: - await session.close() - - # API Key Management - - async def get_api_key(self, key: str) -> Optional[APIKey]: - """Get API key by value.""" - async with self.get_session() as session: - result = await session.execute( - select(APIKey).where( - and_(APIKey.key == key, APIKey.is_active == True) - ) - ) - api_key = result.scalar_one_or_none() - - # Update last used timestamp - if api_key: - api_key.last_used_at = datetime.utcnow() - await session.commit() - - return api_key - - async def create_api_key(self, name: str, description: str = "") -> APIKey: - """Create new API key.""" - api_key = APIKey( - key=str(uuid.uuid4()), - name=name, - description=description - ) - - async with self.get_session() as session: - session.add(api_key) - await session.commit() - await session.refresh(api_key) - - logger.info("api_key_created", key_id=api_key.id, name=name) - return api_key - - async def deactivate_api_key(self, key: str) -> bool: - """Deactivate an API key.""" - async with self.get_session() as session: - result = await session.execute( - select(APIKey).where(APIKey.key == key) - ) - api_key = result.scalar_one_or_none() - - if api_key: - api_key.is_active = False - await session.commit() - logger.info("api_key_deactivated", key_id=api_key.id) - return True - - return False - - # Search Request Logging - - async def log_search_request( - self, - request_data: Dict[str, Any], - response: Optional[UnQuestResponse] = None, - api_key_id: Optional[int] = None, - client_ip: Optional[str] = None, - user_agent: Optional[str] = None - ) -> SearchRequest: - """Log a search request for analytics.""" - search_request = SearchRequest( - request_id=request_data.get("request_id", str(uuid.uuid4())), - api_key_id=api_key_id, - query=request_data["query"], - engines=request_data["engines"], - max_results=request_data["max_results"], - language=request_data.get("language", "en"), - safe_search=request_data.get("safe_search", "moderate"), - client_ip=client_ip, - user_agent=user_agent, - request_headers=request_data.get("headers", {}) - ) - - if response: - search_request.search_time_ms = response.search_metadata.search_time_ms - search_request.total_time_ms = response.processing_time_ms - search_request.results_count = len(response.results) - search_request.scraped_count = sum( - 1 for r in response.results if r.scraped_content - ) - search_request.cache_hit = response.cached - search_request.cache_key = response.cache_key - search_request.completed_at = datetime.utcnow() - - async with self.get_session() as session: - session.add(search_request) - await session.commit() - await session.refresh(search_request) - - return search_request - - async def get_search_analytics( - self, - start_date: Optional[datetime] = None, - end_date: Optional[datetime] = None, - api_key_id: Optional[int] = None - ) -> Dict[str, Any]: - """Get search analytics for the specified period.""" - if not start_date: - start_date = datetime.utcnow() - timedelta(days=7) - if not end_date: - end_date = datetime.utcnow() - - async with self.get_session() as session: - # Base query - base_query = select(SearchRequest).where( - and_( - SearchRequest.created_at >= start_date, - SearchRequest.created_at <= end_date - ) - ) - - if api_key_id: - base_query = base_query.where(SearchRequest.api_key_id == api_key_id) - - # Get total requests - total_result = await session.execute( - select(func.count(SearchRequest.id)).where( - and_( - SearchRequest.created_at >= start_date, - SearchRequest.created_at <= end_date - ) - ) - ) - total_requests = total_result.scalar() or 0 - - # Get cache statistics - cache_result = await session.execute( - select( - func.count(SearchRequest.id).filter(SearchRequest.cache_hit == True) - ).where( - and_( - SearchRequest.created_at >= start_date, - SearchRequest.created_at <= end_date - ) - ) - ) - cache_hits = cache_result.scalar() or 0 - - # Get average response times - timing_result = await session.execute( - select( - func.avg(SearchRequest.search_time_ms), - func.avg(SearchRequest.total_time_ms) - ).where( - and_( - SearchRequest.created_at >= start_date, - SearchRequest.created_at <= end_date, - SearchRequest.search_time_ms.isnot(None) - ) - ) - ) - avg_search_time, avg_total_time = timing_result.one() - - # Get top queries - top_queries_result = await session.execute( - select( - SearchRequest.query, - func.count(SearchRequest.id).label('count') - ).where( - and_( - SearchRequest.created_at >= start_date, - SearchRequest.created_at <= end_date - ) - ).group_by(SearchRequest.query) - .order_by(desc('count')) - .limit(10) - ) - top_queries = [ - {"query": row[0], "count": row[1]} - for row in top_queries_result - ] - - # Get engine usage - engine_stats = {} - requests_with_engines = await session.execute( - select(SearchRequest.engines).where( - and_( - SearchRequest.created_at >= start_date, - SearchRequest.created_at <= end_date - ) - ) - ) - - for row in requests_with_engines: - engines = row[0] - if engines: - for engine in engines: - engine_stats[engine] = engine_stats.get(engine, 0) + 1 - - return { - "period": { - "start": start_date.isoformat(), - "end": end_date.isoformat() - }, - "total_requests": total_requests, - "cache_hits": cache_hits, - "cache_hit_rate": (cache_hits / total_requests * 100) if total_requests > 0 else 0, - "avg_search_time_ms": float(avg_search_time) if avg_search_time else 0, - "avg_total_time_ms": float(avg_total_time) if avg_total_time else 0, - "top_queries": top_queries, - "engine_usage": engine_stats - } - - # Scraping Job Management - - async def create_scraping_job( - self, - urls: List[str], - config: Dict[str, Any], - webhook_url: Optional[str] = None - ) -> ScrapingJob: - """Create a new scraping job.""" - job = ScrapingJob( - urls=urls, - config=config, - webhook_url=webhook_url - ) - - async with self.get_session() as session: - session.add(job) - await session.commit() - await session.refresh(job) - - logger.info("scraping_job_created", job_id=job.job_id) - return job - - async def update_scraping_job( - self, - job_id: str, - status: str, - results: Optional[List[Dict]] = None, - error_message: Optional[str] = None, - task_id: Optional[str] = None - ) -> Optional[ScrapingJob]: - """Update scraping job status.""" - async with self.get_session() as session: - result = await session.execute( - select(ScrapingJob).where(ScrapingJob.job_id == job_id) - ) - job = result.scalar_one_or_none() - - if job: - job.status = status - job.task_id = task_id - - if status == "processing": - job.started_at = datetime.utcnow() - elif status in ["completed", "failed"]: - job.completed_at = datetime.utcnow() - - if results: - job.results = results - if error_message: - job.error_message = error_message - - await session.commit() - logger.info("scraping_job_updated", job_id=job_id, status=status) - - return job - - async def get_scraping_job(self, job_id: str) -> Optional[ScrapingJob]: - """Get scraping job by ID.""" - async with self.get_session() as session: - result = await session.execute( - select(ScrapingJob).where(ScrapingJob.job_id == job_id) - ) - return result.scalar_one_or_none() - - # Error Logging - - async def log_error( - self, - error_type: str, - error_message: str, - request_id: Optional[str] = None, - error_details: Optional[Dict] = None, - stack_trace: Optional[str] = None, - endpoint: Optional[str] = None, - method: Optional[str] = None, - status_code: Optional[int] = None, - client_ip: Optional[str] = None - ): - """Log an error for debugging.""" - error_log = ErrorLog( - request_id=request_id, - error_type=error_type, - error_message=error_message, - error_details=error_details or {}, - stack_trace=stack_trace, - endpoint=endpoint, - method=method, - status_code=status_code, - client_ip=client_ip - ) - - async with self.get_session() as session: - session.add(error_log) - await session.commit() - - logger.error( - "error_logged", - error_type=error_type, - request_id=request_id, - endpoint=endpoint - ) - - async def get_recent_errors( - self, - limit: int = 100, - error_type: Optional[str] = None - ) -> List[ErrorLog]: - """Get recent errors for monitoring.""" - async with self.get_session() as session: - query = select(ErrorLog) - - if error_type: - query = query.where(ErrorLog.error_type == error_type) - - query = query.order_by(desc(ErrorLog.created_at)).limit(limit) - - result = await session.execute(query) - return result.scalars().all() - - -# Singleton instance -_database_service: Optional[DatabaseService] = None - - -async def get_database_service() -> DatabaseService: - """Get or create database service instance.""" - global _database_service - - if _database_service is None: - _database_service = DatabaseService() - await _database_service.initialize() - - return _database_service diff --git a/apps/backend/app/services/database_manager.py b/apps/backend/app/services/database_manager.py deleted file mode 100644 index dff6541..0000000 --- a/apps/backend/app/services/database_manager.py +++ /dev/null @@ -1,653 +0,0 @@ -""" -Advanced database management system with connection pooling, migrations, and caching. - -This module provides comprehensive database management: -- Async connection pooling with SQLite and PostgreSQL support -- Migration system with version management -- Content hashing and deduplication -- Structured logging and error handling -- Performance optimization with batch operations -""" - -import os -import asyncio -import time -import hashlib -from pathlib import Path -from typing import Dict, List, Optional, Any, Union, Tuple -from contextlib import asynccontextmanager -from dataclasses import dataclass, field -from datetime import datetime, timedelta - -import aiosqlite -import structlog - -from app.models.requests import UnQuestRequest, ScrapingConfig - -logger = structlog.get_logger(__name__) - - -@dataclass -class CrawlRecord: - """Record for crawled content storage.""" - url: str - content_hash: str - content: str - metadata: Dict[str, Any] - timestamp: datetime = field(default_factory=datetime.utcnow) - success: bool = True - error: Optional[str] = None - response_code: Optional[int] = None - content_type: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary for database storage.""" - return { - 'url': self.url, - 'content_hash': self.content_hash, - 'content': self.content, - 'metadata': self.metadata, - 'timestamp': self.timestamp.isoformat(), - 'success': self.success, - 'error': self.error, - 'response_code': self.response_code, - 'content_type': self.content_type - } - - -@dataclass -class DatabaseStats: - """Database performance and usage statistics.""" - total_records: int = 0 - successful_crawls: int = 0 - failed_crawls: int = 0 - unique_domains: int = 0 - avg_content_size: float = 0.0 - last_cleanup: Optional[datetime] = None - cache_hit_rate: float = 0.0 - - @property - def success_rate(self) -> float: - """Calculate success rate percentage.""" - if self.total_records == 0: - return 0.0 - return (self.successful_crawls / self.total_records) * 100 - - -class DatabaseManager: - """ - Advanced database manager with connection pooling and performance optimization. - - Features: - - Async connection pooling - - Content deduplication with hashing - - Migration system - - Performance metrics - - Configurable retention policies - """ - - def __init__(self, - db_path: Optional[str] = None, - pool_size: int = 10, - max_retries: int = 3, - retention_days: int = 30): - """ - Initialize database manager. - - Args: - db_path: Database file path (defaults to ~/.unsearch/crawl_data.db) - pool_size: Maximum number of connections in pool - max_retries: Maximum retry attempts for failed operations - retention_days: Days to retain crawl data - """ - # Database configuration - if db_path: - self.db_path = Path(db_path) - else: - base_dir = Path.home() / '.unsearch' - base_dir.mkdir(exist_ok=True) - self.db_path = base_dir / 'crawl_data.db' - - self.pool_size = pool_size - self.max_retries = max_retries - self.retention_days = retention_days - - # Connection management - self.connection_pool: List[aiosqlite.Connection] = [] - self.pool_lock = asyncio.Lock() - self.init_lock = asyncio.Lock() - self.connection_semaphore = asyncio.Semaphore(pool_size) - self._initialized = False - - # Performance tracking - self._stats = DatabaseStats() - self._cache_hits = 0 - self._cache_misses = 0 - - # Schema version for migrations - self.schema_version = 1 - - async def initialize(self): - """Initialize database and connection pool.""" - async with self.init_lock: - if self._initialized: - return - - logger.info(f"Initializing database at {self.db_path}") - - # Ensure database directory exists - self.db_path.parent.mkdir(parents=True, exist_ok=True) - - # Create schema - await self._create_schema() - - # Run migrations if needed - await self._run_migrations() - - # Initialize connection pool - await self._initialize_pool() - - # Update stats - await self._update_stats() - - self._initialized = True - logger.success("Database initialization completed") - - async def _create_schema(self): - """Create database schema.""" - schema_sql = """ - CREATE TABLE IF NOT EXISTS crawl_data ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT NOT NULL, - content_hash TEXT NOT NULL UNIQUE, - content TEXT NOT NULL, - metadata TEXT DEFAULT '{}', - timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, - success BOOLEAN DEFAULT TRUE, - error TEXT NULL, - response_code INTEGER NULL, - content_type TEXT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - - CREATE INDEX IF NOT EXISTS idx_url ON crawl_data(url); - CREATE INDEX IF NOT EXISTS idx_content_hash ON crawl_data(content_hash); - CREATE INDEX IF NOT EXISTS idx_timestamp ON crawl_data(timestamp); - CREATE INDEX IF NOT EXISTS idx_success ON crawl_data(success); - - CREATE TABLE IF NOT EXISTS schema_version ( - version INTEGER PRIMARY KEY, - applied_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - - CREATE TABLE IF NOT EXISTS crawl_stats ( - id INTEGER PRIMARY KEY, - key TEXT UNIQUE NOT NULL, - value TEXT NOT NULL, - updated_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - """ - - async with aiosqlite.connect(self.db_path) as conn: - await conn.executescript(schema_sql) - await conn.commit() - - async def _run_migrations(self): - """Run database migrations.""" - async with aiosqlite.connect(self.db_path) as conn: - # Check current version - cursor = await conn.execute("SELECT MAX(version) FROM schema_version") - result = await cursor.fetchone() - current_version = result[0] if result[0] else 0 - - # Apply migrations if needed - if current_version < self.schema_version: - logger.info(f"Applying migrations from version {current_version} to {self.schema_version}") - - # Example migration (add new columns, indexes, etc.) - if current_version < 1: - await conn.execute(""" - ALTER TABLE crawl_data - ADD COLUMN content_length INTEGER DEFAULT 0 - """) - - # Update schema version - await conn.execute( - "INSERT OR REPLACE INTO schema_version (version) VALUES (?)", - (self.schema_version,) - ) - await conn.commit() - - logger.success(f"Migration to version {self.schema_version} completed") - - async def _initialize_pool(self): - """Initialize connection pool.""" - async with self.pool_lock: - for _ in range(self.pool_size): - conn = await aiosqlite.connect(self.db_path) - # Enable WAL mode for better concurrent access - await conn.execute("PRAGMA journal_mode=WAL") - await conn.execute("PRAGMA synchronous=NORMAL") - await conn.execute("PRAGMA cache_size=10000") - await conn.execute("PRAGMA temp_store=memory") - self.connection_pool.append(conn) - - @asynccontextmanager - async def get_connection(self): - """Get connection from pool.""" - if not self._initialized: - await self.initialize() - - async with self.connection_semaphore: - async with self.pool_lock: - if self.connection_pool: - conn = self.connection_pool.pop() - else: - # Create new connection if pool is empty - conn = await aiosqlite.connect(self.db_path) - - try: - yield conn - finally: - async with self.pool_lock: - if len(self.connection_pool) < self.pool_size: - self.connection_pool.append(conn) - else: - await conn.close() - - def _generate_content_hash(self, content: str, url: str) -> str: - """Generate content hash for deduplication.""" - # Combine URL and content for hash - combined = f"{url}:{content}" - return hashlib.sha256(combined.encode()).hexdigest()[:16] - - async def store_crawl_result(self, - url: str, - content: str, - metadata: Dict[str, Any] = None, - success: bool = True, - error: Optional[str] = None, - response_code: Optional[int] = None, - content_type: Optional[str] = None) -> str: - """ - Store crawl result in database. - - Args: - url: The crawled URL - content: The extracted content - metadata: Additional metadata - success: Whether the crawl was successful - error: Error message if failed - response_code: HTTP response code - content_type: Content type header - - Returns: - Content hash of the stored record - """ - if not self._initialized: - await self.initialize() - - content_hash = self._generate_content_hash(content, url) - metadata = metadata or {} - - record = CrawlRecord( - url=url, - content_hash=content_hash, - content=content, - metadata=metadata, - success=success, - error=error, - response_code=response_code, - content_type=content_type - ) - - for attempt in range(self.max_retries): - try: - async with self.get_connection() as conn: - await conn.execute(""" - INSERT OR REPLACE INTO crawl_data - (url, content_hash, content, metadata, success, error, response_code, content_type) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, ( - record.url, - record.content_hash, - record.content, - str(record.metadata), - record.success, - record.error, - record.response_code, - record.content_type - )) - await conn.commit() - - logger.debug(f"Stored crawl result for {url} with hash {content_hash}") - return content_hash - - except Exception as e: - logger.warning(f"Attempt {attempt + 1} failed to store crawl result: {str(e)}") - if attempt == self.max_retries - 1: - logger.error(f"Failed to store crawl result after {self.max_retries} attempts: {str(e)}") - raise - await asyncio.sleep(0.1 * (attempt + 1)) - - async def get_crawl_result(self, - url: str = None, - content_hash: str = None) -> Optional[CrawlRecord]: - """ - Retrieve crawl result by URL or content hash. - - Args: - url: URL to search for - content_hash: Content hash to search for - - Returns: - CrawlRecord if found, None otherwise - """ - if not self._initialized: - await self.initialize() - - if not url and not content_hash: - raise ValueError("Either url or content_hash must be provided") - - try: - async with self.get_connection() as conn: - if content_hash: - cursor = await conn.execute( - "SELECT * FROM crawl_data WHERE content_hash = ? ORDER BY timestamp DESC LIMIT 1", - (content_hash,) - ) - self._cache_hits += 1 - else: - cursor = await conn.execute( - "SELECT * FROM crawl_data WHERE url = ? ORDER BY timestamp DESC LIMIT 1", - (url,) - ) - self._cache_misses += 1 - - row = await cursor.fetchone() - if row: - return CrawlRecord( - url=row[1], - content_hash=row[2], - content=row[3], - metadata=eval(row[4]) if row[4] else {}, - timestamp=datetime.fromisoformat(row[5]), - success=bool(row[6]), - error=row[7], - response_code=row[8], - content_type=row[9] - ) - - return None - - except Exception as e: - logger.error(f"Failed to retrieve crawl result: {str(e)}") - return None - - async def search_crawl_results(self, - query: str, - limit: int = 100, - offset: int = 0) -> List[CrawlRecord]: - """ - Search crawl results by content or URL. - - Args: - query: Search query - limit: Maximum results to return - offset: Offset for pagination - - Returns: - List of matching CrawlRecord objects - """ - if not self._initialized: - await self.initialize() - - try: - async with self.get_connection() as conn: - cursor = await conn.execute(""" - SELECT * FROM crawl_data - WHERE url LIKE ? OR content LIKE ? - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (f"%{query}%", f"%{query}%", limit, offset)) - - rows = await cursor.fetchall() - results = [] - - for row in rows: - results.append(CrawlRecord( - url=row[1], - content_hash=row[2], - content=row[3], - metadata=eval(row[4]) if row[4] else {}, - timestamp=datetime.fromisoformat(row[5]), - success=bool(row[6]), - error=row[7], - response_code=row[8], - content_type=row[9] - )) - - return results - - except Exception as e: - logger.error(f"Failed to search crawl results: {str(e)}") - return [] - - async def cleanup_old_records(self, days: Optional[int] = None) -> int: - """ - Clean up old crawl records. - - Args: - days: Number of days to retain (defaults to self.retention_days) - - Returns: - Number of records deleted - """ - if not self._initialized: - await self.initialize() - - days = days or self.retention_days - cutoff_date = datetime.utcnow() - timedelta(days=days) - - try: - async with self.get_connection() as conn: - cursor = await conn.execute( - "DELETE FROM crawl_data WHERE timestamp < ?", - (cutoff_date.isoformat(),) - ) - await conn.commit() - - deleted_count = cursor.rowcount - logger.info(f"Cleaned up {deleted_count} records older than {days} days") - - # Update stats - await self._update_stats() - - return deleted_count - - except Exception as e: - logger.error(f"Failed to cleanup old records: {str(e)}") - return 0 - - async def _update_stats(self): - """Update database statistics.""" - try: - async with self.get_connection() as conn: - # Total records - cursor = await conn.execute("SELECT COUNT(*) FROM crawl_data") - self._stats.total_records = (await cursor.fetchone())[0] - - # Successful crawls - cursor = await conn.execute("SELECT COUNT(*) FROM crawl_data WHERE success = 1") - self._stats.successful_crawls = (await cursor.fetchone())[0] - - # Failed crawls - self._stats.failed_crawls = self._stats.total_records - self._stats.successful_crawls - - # Unique domains - cursor = await conn.execute(""" - SELECT COUNT(DISTINCT - CASE - WHEN url LIKE 'http%' THEN - substr(url, instr(url, '://') + 3, instr(substr(url, instr(url, '://') + 3), '/') - 1) - ELSE url - END - ) FROM crawl_data - """) - self._stats.unique_domains = (await cursor.fetchone())[0] - - # Average content size - cursor = await conn.execute("SELECT AVG(LENGTH(content)) FROM crawl_data") - result = await cursor.fetchone() - self._stats.avg_content_size = result[0] if result[0] else 0.0 - - # Cache hit rate - total_requests = self._cache_hits + self._cache_misses - if total_requests > 0: - self._stats.cache_hit_rate = (self._cache_hits / total_requests) * 100 - - self._stats.last_cleanup = datetime.utcnow() - - except Exception as e: - logger.error(f"Failed to update stats: {str(e)}") - - async def get_stats(self) -> DatabaseStats: - """Get current database statistics.""" - if not self._initialized: - await self.initialize() - - await self._update_stats() - return self._stats - - async def get_domain_stats(self, limit: int = 10) -> List[Tuple[str, int]]: - """ - Get statistics by domain. - - Args: - limit: Number of top domains to return - - Returns: - List of (domain, count) tuples - """ - if not self._initialized: - await self.initialize() - - try: - async with self.get_connection() as conn: - cursor = await conn.execute(""" - SELECT - CASE - WHEN url LIKE 'http%' THEN - substr(url, instr(url, '://') + 3, instr(substr(url, instr(url, '://') + 3), '/') - 1) - ELSE url - END as domain, - COUNT(*) as count - FROM crawl_data - GROUP BY domain - ORDER BY count DESC - LIMIT ? - """, (limit,)) - - return await cursor.fetchall() - - except Exception as e: - logger.error(f"Failed to get domain stats: {str(e)}") - return [] - - async def export_data(self, output_path: str, format: str = 'json') -> bool: - """ - Export crawl data to file. - - Args: - output_path: Output file path - format: Export format ('json', 'csv') - - Returns: - True if export successful - """ - if not self._initialized: - await self.initialize() - - try: - async with self.get_connection() as conn: - cursor = await conn.execute("SELECT * FROM crawl_data ORDER BY timestamp DESC") - rows = await cursor.fetchall() - - if format.lower() == 'json': - import json - data = [] - for row in rows: - record = { - 'id': row[0], - 'url': row[1], - 'content_hash': row[2], - 'content': row[3], - 'metadata': row[4], - 'timestamp': row[5], - 'success': bool(row[6]), - 'error': row[7], - 'response_code': row[8], - 'content_type': row[9] - } - data.append(record) - - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - elif format.lower() == 'csv': - import csv - with open(output_path, 'w', newline='', encoding='utf-8') as f: - writer = csv.writer(f) - writer.writerow(['id', 'url', 'content_hash', 'content', 'metadata', - 'timestamp', 'success', 'error', 'response_code', 'content_type']) - writer.writerows(rows) - - logger.success(f"Exported {len(rows)} records to {output_path}") - return True - - except Exception as e: - logger.error(f"Failed to export data: {str(e)}") - return False - - async def close(self): - """Close all connections and cleanup.""" - async with self.pool_lock: - for conn in self.connection_pool: - await conn.close() - self.connection_pool.clear() - - logger.info("Database manager closed") - - -# Singleton instance -_db_manager: Optional[DatabaseManager] = None - - -def get_database_manager(**kwargs) -> DatabaseManager: - """Get singleton database manager instance.""" - global _db_manager - if _db_manager is None: - _db_manager = DatabaseManager(**kwargs) - return _db_manager - - -# Convenience functions -async def store_crawl_data(url: str, content: str, **kwargs) -> str: - """Store crawl data using global database manager.""" - db_manager = get_database_manager() - return await db_manager.store_crawl_result(url, content, **kwargs) - - -async def get_cached_content(url: str) -> Optional[str]: - """Get cached content by URL.""" - db_manager = get_database_manager() - record = await db_manager.get_crawl_result(url=url) - return record.content if record and record.success else None - - -async def search_content(query: str, limit: int = 50) -> List[Dict[str, Any]]: - """Search crawled content.""" - db_manager = get_database_manager() - records = await db_manager.search_crawl_results(query, limit) - return [record.to_dict() for record in records] diff --git a/apps/backend/app/services/deep_crawling.py b/apps/backend/app/services/deep_crawling.py deleted file mode 100644 index a5186f9..0000000 --- a/apps/backend/app/services/deep_crawling.py +++ /dev/null @@ -1,803 +0,0 @@ -""" -Advanced deep crawling system with multiple strategies and sophisticated filtering. - -This module provides comprehensive deep crawling capabilities: -- Multi-strategy crawling (BFS, DFS, Best-First) -- Advanced URL filtering chain -- Composite URL scoring system -- Content relevance filtering -- SEO-based filtering and prioritization -""" - -import re -import asyncio -import time -import math -from abc import ABC, abstractmethod -from typing import Dict, List, Set, Optional, Any, Tuple, Pattern, Union -from urllib.parse import urljoin, urlparse -from collections import deque, defaultdict -from dataclasses import dataclass, field -from enum import Enum - -import structlog -from bs4 import BeautifulSoup - -from app.utils.text_processing import clean_tokens, calculate_text_quality - -logger = structlog.get_logger(__name__) - - -class CrawlOrder(str, Enum): - """Crawl order strategies.""" - BFS = "bfs" # Breadth-first search - DFS = "dfs" # Depth-first search - BEST_FIRST = "best_first" # Best-first based on scoring - - -@dataclass -class FilterStats: - """Statistics for URL filtering operations.""" - total_urls: int = 0 - passed_urls: int = 0 - rejected_urls: int = 0 - - @property - def pass_rate(self) -> float: - """Calculate pass rate.""" - return self.passed_urls / self.total_urls if self.total_urls > 0 else 0.0 - - -@dataclass -class CrawlProgress: - """Progress tracking for deep crawling operations.""" - urls_discovered: int = 0 - urls_processed: int = 0 - urls_successful: int = 0 - urls_failed: int = 0 - current_depth: int = 0 - start_time: float = field(default_factory=time.time) - - @property - def success_rate(self) -> float: - """Calculate success rate.""" - return self.urls_successful / self.urls_processed if self.urls_processed > 0 else 0.0 - - @property - def elapsed_time(self) -> float: - """Calculate elapsed time.""" - return time.time() - self.start_time - - -# URL Filtering System -class URLFilter(ABC): - """Base class for URL filtering.""" - - def __init__(self, name: str = None): - self.name = name or self.__class__.__name__ - self.stats = FilterStats() - - @abstractmethod - def apply(self, url: str, **kwargs) -> bool: - """Apply filter to URL and return True if URL should be kept.""" - pass - - def _update_stats(self, passed: bool): - """Update filter statistics.""" - self.stats.total_urls += 1 - if passed: - self.stats.passed_urls += 1 - else: - self.stats.rejected_urls += 1 - - -class DomainFilter(URLFilter): - """Filter URLs based on allowed/blocked domains.""" - - def __init__(self, - allowed_domains: List[str] = None, - blocked_domains: List[str] = None, - same_domain_only: bool = False, - base_domain: str = None): - super().__init__() - self.allowed_domains = set(allowed_domains or []) - self.blocked_domains = set(blocked_domains or []) - self.same_domain_only = same_domain_only - self.base_domain = base_domain - - def apply(self, url: str, **kwargs) -> bool: - """Apply domain filtering.""" - try: - parsed = urlparse(url) - domain = parsed.netloc.lower() - - # Same domain only check - if self.same_domain_only and self.base_domain: - if domain != self.base_domain.lower(): - self._update_stats(False) - return False - - # Blocked domains check - if self.blocked_domains: - for blocked in self.blocked_domains: - if blocked.lower() in domain: - self._update_stats(False) - return False - - # Allowed domains check - if self.allowed_domains: - allowed = any(allowed.lower() in domain for allowed in self.allowed_domains) - self._update_stats(allowed) - return allowed - - self._update_stats(True) - return True - - except Exception: - self._update_stats(False) - return False - - -class URLPatternFilter(URLFilter): - """Filter URLs based on regex patterns.""" - - def __init__(self, - include_patterns: List[str] = None, - exclude_patterns: List[str] = None): - super().__init__() - self.include_patterns = [re.compile(p) for p in (include_patterns or [])] - self.exclude_patterns = [re.compile(p) for p in (exclude_patterns or [])] - - def apply(self, url: str, **kwargs) -> bool: - """Apply pattern filtering.""" - # Check exclude patterns first - for pattern in self.exclude_patterns: - if pattern.search(url): - self._update_stats(False) - return False - - # Check include patterns - if self.include_patterns: - for pattern in self.include_patterns: - if pattern.search(url): - self._update_stats(True) - return True - self._update_stats(False) - return False - - self._update_stats(True) - return True - - -class ContentTypeFilter(URLFilter): - """Filter URLs based on expected content type.""" - - def __init__(self, - allowed_types: List[str] = None, - blocked_types: List[str] = None): - super().__init__() - self.allowed_types = set(allowed_types or ['text/html', 'application/xhtml+xml']) - self.blocked_types = set(blocked_types or []) - - # Common file extensions to block for web content - self.blocked_extensions = { - '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', - '.zip', '.tar', '.gz', '.rar', '.7z', - '.mp4', '.avi', '.mov', '.wmv', '.flv', - '.mp3', '.wav', '.flac', '.aac', - '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp' - } - - def apply(self, url: str, **kwargs) -> bool: - """Apply content type filtering.""" - parsed = urlparse(url) - path = parsed.path.lower() - - # Check file extensions - for ext in self.blocked_extensions: - if path.endswith(ext): - self._update_stats(False) - return False - - # If content type is provided in kwargs, check it - content_type = kwargs.get('content_type', '').lower() - if content_type: - if self.blocked_types and any(blocked in content_type for blocked in self.blocked_types): - self._update_stats(False) - return False - - if self.allowed_types and not any(allowed in content_type for allowed in self.allowed_types): - self._update_stats(False) - return False - - self._update_stats(True) - return True - - -class SEOFilter(URLFilter): - """Filter URLs based on SEO and quality indicators.""" - - def __init__(self, max_depth: int = 5, max_params: int = 10): - super().__init__() - self.max_depth = max_depth - self.max_params = max_params - - # SEO-unfriendly patterns - self.bad_patterns = [ - r'/cgi-bin/', r'/admin/', r'/private/', r'/tmp/', - r'\?.*sessionid', r'\?.*sid=', r'\?.*PHPSESSID', - r'\.php\?.*&.*&.*&', r'javascript:', r'mailto:', - ] - self.bad_regexes = [re.compile(pattern, re.IGNORECASE) for pattern in self.bad_patterns] - - def apply(self, url: str, **kwargs) -> bool: - """Apply SEO filtering.""" - parsed = urlparse(url) - - # Check URL depth - path_parts = [p for p in parsed.path.split('/') if p] - if len(path_parts) > self.max_depth: - self._update_stats(False) - return False - - # Check number of parameters - if parsed.query: - params = parsed.query.split('&') - if len(params) > self.max_params: - self._update_stats(False) - return False - - # Check for bad patterns - for regex in self.bad_regexes: - if regex.search(url): - self._update_stats(False) - return False - - self._update_stats(True) - return True - - -class ContentRelevanceFilter(URLFilter): - """Filter URLs based on content relevance to a query.""" - - def __init__(self, query: str = None, min_score: float = 0.3): - super().__init__() - self.query_tokens = set(clean_tokens(query.lower().split())) if query else set() - self.min_score = min_score - - def apply(self, url: str, **kwargs) -> bool: - """Apply content relevance filtering.""" - if not self.query_tokens: - self._update_stats(True) - return True - - # Score URL based on query relevance - url_lower = url.lower() - url_tokens = set(re.findall(r'[a-zA-Z]+', url_lower)) - - if not url_tokens: - self._update_stats(False) - return False - - # Calculate overlap score - overlap = len(self.query_tokens.intersection(url_tokens)) - score = overlap / len(self.query_tokens) if self.query_tokens else 0.0 - - passed = score >= self.min_score - self._update_stats(passed) - return passed - - -class FilterChain: - """Chain multiple URL filters together.""" - - def __init__(self, filters: List[URLFilter] = None): - self.filters = filters or [] - self.stats = FilterStats() - - def add_filter(self, filter_obj: URLFilter): - """Add a filter to the chain.""" - self.filters.append(filter_obj) - - def apply(self, url: str, **kwargs) -> bool: - """Apply all filters in sequence.""" - self.stats.total_urls += 1 - - for filter_obj in self.filters: - if not filter_obj.apply(url, **kwargs): - self.stats.rejected_urls += 1 - return False - - self.stats.passed_urls += 1 - return True - - -# URL Scoring System -class URLScorer(ABC): - """Base class for URL scoring.""" - - def __init__(self, weight: float = 1.0): - self.weight = weight - - @abstractmethod - def calculate_score(self, url: str, **kwargs) -> float: - """Calculate score for URL (0.0 to 1.0).""" - pass - - def score(self, url: str, **kwargs) -> float: - """Calculate weighted score.""" - return self.calculate_score(url, **kwargs) * self.weight - - -class KeywordRelevanceScorer(URLScorer): - """Score URLs based on keyword relevance.""" - - def __init__(self, keywords: List[str], weight: float = 1.0): - super().__init__(weight) - self.keywords = [kw.lower() for kw in keywords] - - def calculate_score(self, url: str, **kwargs) -> float: - """Calculate keyword relevance score.""" - if not self.keywords: - return 0.5 - - url_lower = url.lower() - matches = sum(1 for keyword in self.keywords if keyword in url_lower) - return min(1.0, matches / len(self.keywords)) - - -class PathDepthScorer(URLScorer): - """Score URLs based on path depth (shorter paths = higher score).""" - - def __init__(self, max_depth: int = 5, weight: float = 1.0): - super().__init__(weight) - self.max_depth = max_depth - - def calculate_score(self, url: str, **kwargs) -> float: - """Calculate depth score.""" - parsed = urlparse(url) - depth = len([p for p in parsed.path.split('/') if p]) - return max(0.0, 1.0 - (depth / self.max_depth)) - - -class DomainAuthorityScorer(URLScorer): - """Score URLs based on domain authority.""" - - def __init__(self, weight: float = 1.0): - super().__init__(weight) - # Predefined domain authority scores - self.domain_scores = { - # High authority domains - 'wikipedia.org': 1.0, - 'github.com': 0.9, - 'stackoverflow.com': 0.9, - 'mozilla.org': 0.8, - 'w3.org': 0.8, - # Medium authority domains - 'medium.com': 0.7, - 'dev.to': 0.6, - 'reddit.com': 0.6, - } - - def calculate_score(self, url: str, **kwargs) -> float: - """Calculate domain authority score.""" - parsed = urlparse(url) - domain = parsed.netloc.lower() - - # Check for exact matches - if domain in self.domain_scores: - return self.domain_scores[domain] - - # Check for subdomain matches - for scored_domain, score in self.domain_scores.items(): - if domain.endswith('.' + scored_domain): - return score * 0.8 # Subdomains get 80% of parent score - - return 0.5 # Default score for unknown domains - - -class FreshnessScorer(URLScorer): - """Score URLs based on estimated freshness.""" - - def __init__(self, weight: float = 1.0): - super().__init__(weight) - self.current_year = time.gmtime().tm_year - - def calculate_score(self, url: str, **kwargs) -> float: - """Calculate freshness score based on URL indicators.""" - # Look for year patterns in URL - years = re.findall(r'\b(20[0-9]{2})\b', url) - if years: - latest_year = max(int(year) for year in years) - years_old = self.current_year - latest_year - return max(0.0, 1.0 - (years_old / 10.0)) # Decay over 10 years - - return 0.5 # Default score if no year found - - -class CompositeScorer(URLScorer): - """Combine multiple scorers with weighted average.""" - - def __init__(self, scorers: List[URLScorer], normalize: bool = True): - # Calculate total weight - total_weight = sum(scorer.weight for scorer in scorers) - super().__init__(total_weight if not normalize else 1.0) - self.scorers = scorers - self.normalize = normalize - - def calculate_score(self, url: str, **kwargs) -> float: - """Calculate composite score.""" - if not self.scorers: - return 0.5 - - total_score = sum(scorer.score(url, **kwargs) for scorer in self.scorers) - - if self.normalize: - total_weight = sum(scorer.weight for scorer in self.scorers) - return total_score / total_weight if total_weight > 0 else 0.0 - else: - return total_score - - -# Deep Crawling Strategies -class DeepCrawlStrategy(ABC): - """Base class for deep crawling strategies.""" - - def __init__(self, - max_depth: int = 3, - max_pages: int = 100, - filter_chain: FilterChain = None, - url_scorer: URLScorer = None): - self.max_depth = max_depth - self.max_pages = max_pages - self.filter_chain = filter_chain or FilterChain() - self.url_scorer = url_scorer - self.progress = CrawlProgress() - - # Crawling state - self.discovered_urls: Set[str] = set() - self.processed_urls: Set[str] = set() - self.url_to_depth: Dict[str, int] = {} - - @abstractmethod - async def crawl(self, - start_urls: List[str], - fetch_callback, - extract_links_callback) -> List[Dict[str, Any]]: - """Execute the crawling strategy.""" - pass - - def _extract_links_from_html(self, html: str, base_url: str) -> List[str]: - """Extract links from HTML content.""" - try: - soup = BeautifulSoup(html, 'lxml') - links = [] - - for anchor in soup.find_all('a', href=True): - href = anchor['href'].strip() - if href: - # Convert relative URLs to absolute - absolute_url = urljoin(base_url, href) - links.append(absolute_url) - - return links - - except Exception as e: - logger.warning(f"Error extracting links from {base_url}: {str(e)}") - return [] - - -class BFSDeepCrawlStrategy(DeepCrawlStrategy): - """Breadth-first search deep crawling strategy.""" - - async def crawl(self, - start_urls: List[str], - fetch_callback, - extract_links_callback) -> List[Dict[str, Any]]: - """Execute BFS crawling.""" - results = [] - queue = deque() - - # Initialize with start URLs - for url in start_urls: - queue.append((url, 0)) # (url, depth) - self.discovered_urls.add(url) - self.url_to_depth[url] = 0 - - self.progress.urls_discovered = len(queue) - - while queue and len(self.processed_urls) < self.max_pages: - current_url, depth = queue.popleft() - - if current_url in self.processed_urls or depth > self.max_depth: - continue - - self.processed_urls.add(current_url) - self.progress.urls_processed += 1 - self.progress.current_depth = depth - - try: - # Fetch page content - page_result = await fetch_callback(current_url) - if page_result: - results.append({ - 'url': current_url, - 'depth': depth, - 'content': page_result, - 'timestamp': time.time() - }) - self.progress.urls_successful += 1 - - # Extract links for next level - if depth < self.max_depth: - extracted_links = await extract_links_callback( - page_result.get('html', ''), current_url - ) - - for link in extracted_links: - if (link not in self.discovered_urls and - self.filter_chain.apply(link)): - - queue.append((link, depth + 1)) - self.discovered_urls.add(link) - self.url_to_depth[link] = depth + 1 - self.progress.urls_discovered += 1 - - except Exception as e: - logger.error(f"Error processing {current_url}: {str(e)}") - self.progress.urls_failed += 1 - - logger.info( - f"BFS crawl completed: {len(results)} pages processed, " - f"success rate: {self.progress.success_rate:.2%}" - ) - - return results - - -class DFSDeepCrawlStrategy(DeepCrawlStrategy): - """Depth-first search deep crawling strategy.""" - - async def crawl(self, - start_urls: List[str], - fetch_callback, - extract_links_callback) -> List[Dict[str, Any]]: - """Execute DFS crawling.""" - results = [] - - for start_url in start_urls: - if len(self.processed_urls) >= self.max_pages: - break - - await self._dfs_recursive( - start_url, 0, results, fetch_callback, extract_links_callback - ) - - logger.info( - f"DFS crawl completed: {len(results)} pages processed, " - f"success rate: {self.progress.success_rate:.2%}" - ) - - return results - - async def _dfs_recursive(self, - url: str, - depth: int, - results: List[Dict[str, Any]], - fetch_callback, - extract_links_callback): - """Recursive DFS implementation.""" - if (url in self.processed_urls or - depth > self.max_depth or - len(self.processed_urls) >= self.max_pages): - return - - self.processed_urls.add(url) - self.progress.urls_processed += 1 - self.progress.current_depth = max(self.progress.current_depth, depth) - - try: - # Fetch page content - page_result = await fetch_callback(url) - if page_result: - results.append({ - 'url': url, - 'depth': depth, - 'content': page_result, - 'timestamp': time.time() - }) - self.progress.urls_successful += 1 - - # Extract and recurse into links - if depth < self.max_depth: - extracted_links = await extract_links_callback( - page_result.get('html', ''), url - ) - - for link in extracted_links: - if (link not in self.discovered_urls and - self.filter_chain.apply(link)): - - self.discovered_urls.add(link) - self.progress.urls_discovered += 1 - - await self._dfs_recursive( - link, depth + 1, results, - fetch_callback, extract_links_callback - ) - - except Exception as e: - logger.error(f"Error processing {url}: {str(e)}") - self.progress.urls_failed += 1 - - -class BestFirstCrawlStrategy(DeepCrawlStrategy): - """Best-first search using URL scoring.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if not self.url_scorer: - # Default composite scorer - self.url_scorer = CompositeScorer([ - KeywordRelevanceScorer(['content', 'article', 'blog'], weight=0.3), - PathDepthScorer(weight=0.2), - DomainAuthorityScorer(weight=0.3), - FreshnessScorer(weight=0.2) - ]) - - async def crawl(self, - start_urls: List[str], - fetch_callback, - extract_links_callback) -> List[Dict[str, Any]]: - """Execute best-first crawling with priority queue.""" - import heapq - - results = [] - # Priority queue: (negative_score, url, depth) - priority_queue = [] - - # Initialize with start URLs - for url in start_urls: - score = self.url_scorer.score(url) - heapq.heappush(priority_queue, (-score, url, 0)) - self.discovered_urls.add(url) - self.url_to_depth[url] = 0 - - self.progress.urls_discovered = len(priority_queue) - - while priority_queue and len(self.processed_urls) < self.max_pages: - negative_score, current_url, depth = heapq.heappop(priority_queue) - - if current_url in self.processed_urls or depth > self.max_depth: - continue - - self.processed_urls.add(current_url) - self.progress.urls_processed += 1 - self.progress.current_depth = depth - - try: - # Fetch page content - page_result = await fetch_callback(current_url) - if page_result: - results.append({ - 'url': current_url, - 'depth': depth, - 'score': -negative_score, - 'content': page_result, - 'timestamp': time.time() - }) - self.progress.urls_successful += 1 - - # Extract and score links - if depth < self.max_depth: - extracted_links = await extract_links_callback( - page_result.get('html', ''), current_url - ) - - for link in extracted_links: - if (link not in self.discovered_urls and - self.filter_chain.apply(link)): - - link_score = self.url_scorer.score(link) - heapq.heappush(priority_queue, (-link_score, link, depth + 1)) - self.discovered_urls.add(link) - self.url_to_depth[link] = depth + 1 - self.progress.urls_discovered += 1 - - except Exception as e: - logger.error(f"Error processing {current_url}: {str(e)}") - self.progress.urls_failed += 1 - - # Sort results by score for best-first - results.sort(key=lambda x: x.get('score', 0), reverse=True) - - logger.info( - f"Best-first crawl completed: {len(results)} pages processed, " - f"success rate: {self.progress.success_rate:.2%}" - ) - - return results - - -# Factory functions -def create_deep_crawl_strategy( - strategy_type: str, - config: Dict[str, Any] = None -) -> DeepCrawlStrategy: - """Create a deep crawling strategy.""" - config = config or {} - - # Create filter chain - filter_chain = FilterChain() - - if config.get('domain_filter'): - filter_chain.add_filter(DomainFilter(**config['domain_filter'])) - - if config.get('pattern_filter'): - filter_chain.add_filter(URLPatternFilter(**config['pattern_filter'])) - - if config.get('content_type_filter'): - filter_chain.add_filter(ContentTypeFilter(**config['content_type_filter'])) - - if config.get('seo_filter'): - filter_chain.add_filter(SEOFilter(**config['seo_filter'])) - - if config.get('relevance_filter'): - filter_chain.add_filter(ContentRelevanceFilter(**config['relevance_filter'])) - - # Create URL scorer for best-first - url_scorer = None - if strategy_type == 'best_first' and config.get('scoring'): - scorers = [] - scoring_config = config['scoring'] - - if scoring_config.get('keyword_relevance'): - scorers.append(KeywordRelevanceScorer(**scoring_config['keyword_relevance'])) - - if scoring_config.get('path_depth'): - scorers.append(PathDepthScorer(**scoring_config['path_depth'])) - - if scoring_config.get('domain_authority'): - scorers.append(DomainAuthorityScorer(**scoring_config['domain_authority'])) - - if scoring_config.get('freshness'): - scorers.append(FreshnessScorer(**scoring_config['freshness'])) - - if scorers: - url_scorer = CompositeScorer(scorers) - - # Create strategy - base_config = { - 'max_depth': config.get('max_depth', 3), - 'max_pages': config.get('max_pages', 100), - 'filter_chain': filter_chain, - 'url_scorer': url_scorer - } - - strategies = { - 'bfs': BFSDeepCrawlStrategy, - 'dfs': DFSDeepCrawlStrategy, - 'best_first': BestFirstCrawlStrategy - } - - if strategy_type not in strategies: - raise ValueError(f"Unknown strategy: {strategy_type}. Available: {list(strategies.keys())}") - - strategy_class = strategies[strategy_type] - return strategy_class(**base_config) - - -# Convenience functions -async def deep_crawl( - start_urls: List[str], - strategy: str = 'bfs', - config: Dict[str, Any] = None, - fetch_callback = None, - extract_links_callback = None -) -> List[Dict[str, Any]]: - """Convenience function for deep crawling.""" - if not fetch_callback or not extract_links_callback: - raise ValueError("fetch_callback and extract_links_callback are required") - - crawler = create_deep_crawl_strategy(strategy, config) - return await crawler.crawl(start_urls, fetch_callback, extract_links_callback) diff --git a/apps/backend/app/services/dispatcher.py b/apps/backend/app/services/dispatcher.py deleted file mode 100644 index b60a2c3..0000000 --- a/apps/backend/app/services/dispatcher.py +++ /dev/null @@ -1,664 +0,0 @@ -""" -Advanced dispatcher system for managing concurrent crawling operations. - -This module provides sophisticated dispatchers for controlling: -- Concurrent request management -- Rate limiting and throttling -- Memory-aware resource allocation -- Adaptive performance optimization -""" - -import asyncio -import time -import psutil -from abc import ABC, abstractmethod -from typing import Dict, List, Optional, Any, Callable, Awaitable -from dataclasses import dataclass, field -from collections import defaultdict, deque - -import structlog - -logger = structlog.get_logger(__name__) - - -@dataclass -class DispatchStats: - """Statistics for dispatcher operations.""" - total_requests: int = 0 - successful_requests: int = 0 - failed_requests: int = 0 - rate_limited_requests: int = 0 - avg_response_time: float = 0.0 - peak_memory_usage: float = 0.0 - current_active: int = 0 - - def update_response_time(self, response_time: float): - """Update average response time with new measurement.""" - if self.successful_requests == 0: - self.avg_response_time = response_time - else: - # Exponential moving average - alpha = 0.1 - self.avg_response_time = (alpha * response_time) + ((1 - alpha) * self.avg_response_time) - - -@dataclass -class TaskResult: - """Result of a dispatched task.""" - task_id: str - success: bool - result: Any = None - error: Optional[str] = None - start_time: float = 0.0 - end_time: float = 0.0 - memory_usage: float = 0.0 - - @property - def duration(self) -> float: - """Get task duration in seconds.""" - return self.end_time - self.start_time if self.end_time > self.start_time else 0.0 - - -class BaseDispatcher(ABC): - """Abstract base class for all dispatchers.""" - - def __init__(self, max_concurrent: int = 10, **kwargs): - """ - Initialize base dispatcher. - - Args: - max_concurrent: Maximum concurrent operations - **kwargs: Additional configuration options - """ - self.max_concurrent = max_concurrent - self.stats = DispatchStats() - self.active_tasks: Dict[str, asyncio.Task] = {} - self.verbose = kwargs.get('verbose', False) - - @abstractmethod - async def dispatch( - self, - tasks: List[Callable[..., Awaitable[Any]]], - *args, - **kwargs - ) -> List[TaskResult]: - """ - Dispatch tasks for execution. - - Args: - tasks: List of async callables to execute - *args: Arguments to pass to each task - **kwargs: Keyword arguments to pass to each task - - Returns: - List of TaskResult objects - """ - pass - - async def cleanup(self): - """Cleanup dispatcher resources.""" - # Cancel any remaining active tasks - for task in self.active_tasks.values(): - if not task.done(): - task.cancel() - - # Wait for tasks to finish cancellation - if self.active_tasks: - await asyncio.gather(*self.active_tasks.values(), return_exceptions=True) - - self.active_tasks.clear() - - -class SemaphoreDispatcher(BaseDispatcher): - """ - Simple semaphore-based dispatcher for basic concurrency control. - - Uses asyncio.Semaphore to limit concurrent operations without - sophisticated rate limiting or memory management. - """ - - def __init__(self, max_concurrent: int = 10, **kwargs): - """Initialize semaphore dispatcher.""" - super().__init__(max_concurrent, **kwargs) - self.semaphore = asyncio.Semaphore(max_concurrent) - - async def dispatch( - self, - tasks: List[Callable[..., Awaitable[Any]]], - *args, - **kwargs - ) -> List[TaskResult]: - """Dispatch tasks with semaphore-based concurrency control.""" - if not tasks: - return [] - - # Create task wrappers - wrapped_tasks = [] - for i, task in enumerate(tasks): - task_id = f"task_{i}_{int(time.time() * 1000)}" - wrapped_task = self._wrap_task(task, task_id, *args, **kwargs) - wrapped_tasks.append(wrapped_task) - - # Execute all tasks - results = await asyncio.gather(*wrapped_tasks, return_exceptions=True) - - # Process results - task_results = [] - for result in results: - if isinstance(result, TaskResult): - task_results.append(result) - elif isinstance(result, Exception): - task_results.append(TaskResult( - task_id=f"error_{int(time.time())}", - success=False, - error=str(result) - )) - else: - task_results.append(TaskResult( - task_id=f"unknown_{int(time.time())}", - success=True, - result=result - )) - - return task_results - - async def _wrap_task( - self, - task: Callable[..., Awaitable[Any]], - task_id: str, - *args, - **kwargs - ) -> TaskResult: - """Wrap task with semaphore and monitoring.""" - async with self.semaphore: - start_time = time.time() - start_memory = psutil.virtual_memory().used - - self.stats.current_active += 1 - self.stats.total_requests += 1 - - try: - result = await task(*args, **kwargs) - - end_time = time.time() - end_memory = psutil.virtual_memory().used - memory_usage = end_memory - start_memory - - self.stats.successful_requests += 1 - self.stats.update_response_time(end_time - start_time) - self.stats.peak_memory_usage = max( - self.stats.peak_memory_usage, - memory_usage - ) - - return TaskResult( - task_id=task_id, - success=True, - result=result, - start_time=start_time, - end_time=end_time, - memory_usage=memory_usage - ) - - except Exception as e: - end_time = time.time() - self.stats.failed_requests += 1 - - logger.error(f"Task {task_id} failed: {str(e)}") - - return TaskResult( - task_id=task_id, - success=False, - error=str(e), - start_time=start_time, - end_time=end_time - ) - - finally: - self.stats.current_active -= 1 - - -class RateLimiter: - """ - Advanced rate limiter with multiple strategies. - - Supports: - - Token bucket algorithm - - Sliding window rate limiting - - Per-domain rate limiting - - Adaptive rate adjustment - """ - - def __init__( - self, - max_requests: int = 100, - time_window: float = 60.0, # seconds - burst_size: Optional[int] = None, - per_domain: bool = False - ): - """ - Initialize rate limiter. - - Args: - max_requests: Maximum requests per time window - time_window: Time window in seconds - burst_size: Maximum burst size (defaults to max_requests) - per_domain: Whether to apply rate limiting per domain - """ - self.max_requests = max_requests - self.time_window = time_window - self.burst_size = burst_size or max_requests - self.per_domain = per_domain - - # Token bucket state - self.tokens = self.burst_size - self.last_refill = time.time() - - # Per-domain limiters - self.domain_limiters: Dict[str, 'RateLimiter'] = {} - - # Request history for sliding window - self.request_history = deque() - - self.lock = asyncio.Lock() - - async def acquire(self, domain: Optional[str] = None) -> bool: - """ - Acquire permission to make a request. - - Args: - domain: Domain for per-domain limiting - - Returns: - True if request is allowed, False if rate limited - """ - async with self.lock: - current_time = time.time() - - # Per-domain rate limiting - if self.per_domain and domain: - if domain not in self.domain_limiters: - self.domain_limiters[domain] = RateLimiter( - max_requests=self.max_requests, - time_window=self.time_window, - burst_size=self.burst_size, - per_domain=False # Avoid infinite recursion - ) - return await self.domain_limiters[domain].acquire() - - # Refill tokens based on elapsed time - self._refill_tokens(current_time) - - # Check if we have tokens available - if self.tokens >= 1: - self.tokens -= 1 - self.request_history.append(current_time) - return True - - return False - - def _refill_tokens(self, current_time: float): - """Refill token bucket based on elapsed time.""" - elapsed = current_time - self.last_refill - - if elapsed > 0: - # Calculate tokens to add - refill_rate = self.max_requests / self.time_window - new_tokens = elapsed * refill_rate - - # Add tokens up to burst size - self.tokens = min(self.burst_size, self.tokens + new_tokens) - self.last_refill = current_time - - def get_wait_time(self) -> float: - """Get estimated wait time before next request can be made.""" - if self.tokens >= 1: - return 0.0 - - # Calculate time needed for one token - refill_rate = self.max_requests / self.time_window - return 1.0 / refill_rate - - def get_stats(self) -> Dict[str, Any]: - """Get rate limiter statistics.""" - current_time = time.time() - - # Clean old requests from history - cutoff_time = current_time - self.time_window - while self.request_history and self.request_history[0] < cutoff_time: - self.request_history.popleft() - - return { - "current_tokens": self.tokens, - "max_tokens": self.burst_size, - "requests_in_window": len(self.request_history), - "max_requests_per_window": self.max_requests, - "time_window": self.time_window, - "estimated_wait_time": self.get_wait_time() - } - - -class MemoryAdaptiveDispatcher(BaseDispatcher): - """ - Memory-aware dispatcher that adapts concurrency based on system resources. - - This dispatcher monitors system memory usage and automatically adjusts - the number of concurrent operations to prevent resource exhaustion. - """ - - def __init__( - self, - max_concurrent: int = 10, - memory_threshold: float = 80.0, # Percentage - rate_limiter: Optional[RateLimiter] = None, - **kwargs - ): - """ - Initialize memory-adaptive dispatcher. - - Args: - max_concurrent: Maximum concurrent operations - memory_threshold: Memory usage threshold percentage (0-100) - rate_limiter: Optional rate limiter instance - """ - super().__init__(max_concurrent, **kwargs) - self.memory_threshold = memory_threshold - self.rate_limiter = rate_limiter or RateLimiter() - - # Adaptive parameters - self.current_max_concurrent = max_concurrent - self.adaptation_history = deque(maxlen=10) - self.last_adaptation = time.time() - - # Performance tracking - self.performance_window = deque(maxlen=100) - - # Create semaphore - self.semaphore = asyncio.Semaphore(self.current_max_concurrent) - - async def dispatch( - self, - tasks: List[Callable[..., Awaitable[Any]]], - *args, - **kwargs - ) -> List[TaskResult]: - """Dispatch tasks with memory-adaptive concurrency control.""" - if not tasks: - return [] - - # Adapt concurrency based on current conditions - await self._adapt_concurrency() - - # Create task wrappers with rate limiting - wrapped_tasks = [] - for i, task in enumerate(tasks): - task_id = f"task_{i}_{int(time.time() * 1000)}" - wrapped_task = self._wrap_task_with_rate_limiting( - task, task_id, *args, **kwargs - ) - wrapped_tasks.append(wrapped_task) - - # Execute all tasks - results = await asyncio.gather(*wrapped_tasks, return_exceptions=True) - - # Process and return results - return self._process_results(results) - - async def _adapt_concurrency(self): - """Adapt concurrency level based on system conditions.""" - current_time = time.time() - - # Only adapt periodically to avoid thrashing - if current_time - self.last_adaptation < 5.0: # 5 second cooldown - return - - memory_usage = psutil.virtual_memory().percent - cpu_usage = psutil.cpu_percent(interval=0.1) - - # Calculate performance score - performance_score = self._calculate_performance_score() - - # Determine adaptation direction - should_increase = ( - memory_usage < self.memory_threshold * 0.7 and - cpu_usage < 70.0 and - performance_score > 0.8 and - self.current_max_concurrent < self.max_concurrent - ) - - should_decrease = ( - memory_usage > self.memory_threshold or - cpu_usage > 90.0 or - performance_score < 0.5 - ) - - old_limit = self.current_max_concurrent - - if should_increase: - self.current_max_concurrent = min( - self.max_concurrent, - int(self.current_max_concurrent * 1.2) - ) - elif should_decrease: - self.current_max_concurrent = max( - 1, - int(self.current_max_concurrent * 0.8) - ) - - # Update semaphore if limit changed - if self.current_max_concurrent != old_limit: - self.semaphore = asyncio.Semaphore(self.current_max_concurrent) - - self.adaptation_history.append({ - 'timestamp': current_time, - 'old_limit': old_limit, - 'new_limit': self.current_max_concurrent, - 'memory_usage': memory_usage, - 'cpu_usage': cpu_usage, - 'performance_score': performance_score - }) - - if self.verbose: - logger.info( - f"Adapted concurrency: {old_limit} -> {self.current_max_concurrent} " - f"(mem: {memory_usage:.1f}%, cpu: {cpu_usage:.1f}%, perf: {performance_score:.2f})" - ) - - self.last_adaptation = current_time - - def _calculate_performance_score(self) -> float: - """Calculate overall performance score (0-1).""" - if not self.performance_window: - return 1.0 - - # Calculate success rate - successful_tasks = sum(1 for p in self.performance_window if p['success']) - success_rate = successful_tasks / len(self.performance_window) - - # Calculate average response time score (lower is better) - avg_response_time = sum(p['response_time'] for p in self.performance_window) / len(self.performance_window) - response_time_score = max(0.0, 1.0 - (avg_response_time / 30.0)) # 30s baseline - - # Calculate memory efficiency score - if self.stats.peak_memory_usage > 0: - memory_score = max(0.0, 1.0 - (self.stats.peak_memory_usage / (1024 * 1024 * 1024))) # 1GB baseline - else: - memory_score = 1.0 - - # Weighted combination - return (success_rate * 0.5) + (response_time_score * 0.3) + (memory_score * 0.2) - - async def _wrap_task_with_rate_limiting( - self, - task: Callable[..., Awaitable[Any]], - task_id: str, - *args, - **kwargs - ) -> TaskResult: - """Wrap task with both semaphore and rate limiting.""" - # Rate limiting check - domain = kwargs.get('domain') or kwargs.get('url', '').split('//')[-1].split('/')[0] if kwargs.get('url') else None - - while not await self.rate_limiter.acquire(domain): - wait_time = self.rate_limiter.get_wait_time() - await asyncio.sleep(max(0.1, wait_time)) - self.stats.rate_limited_requests += 1 - - # Semaphore-based execution - async with self.semaphore: - start_time = time.time() - start_memory = psutil.virtual_memory().used - - self.stats.current_active += 1 - self.stats.total_requests += 1 - - try: - result = await task(*args, **kwargs) - - end_time = time.time() - end_memory = psutil.virtual_memory().used - memory_usage = end_memory - start_memory - response_time = end_time - start_time - - self.stats.successful_requests += 1 - self.stats.update_response_time(response_time) - self.stats.peak_memory_usage = max( - self.stats.peak_memory_usage, - memory_usage - ) - - # Track performance - self.performance_window.append({ - 'success': True, - 'response_time': response_time, - 'memory_usage': memory_usage - }) - - return TaskResult( - task_id=task_id, - success=True, - result=result, - start_time=start_time, - end_time=end_time, - memory_usage=memory_usage - ) - - except Exception as e: - end_time = time.time() - response_time = end_time - start_time - - self.stats.failed_requests += 1 - - # Track performance - self.performance_window.append({ - 'success': False, - 'response_time': response_time, - 'memory_usage': 0 - }) - - logger.error(f"Task {task_id} failed: {str(e)}") - - return TaskResult( - task_id=task_id, - success=False, - error=str(e), - start_time=start_time, - end_time=end_time - ) - - finally: - self.stats.current_active -= 1 - - def _process_results(self, results: List[Any]) -> List[TaskResult]: - """Process raw results into TaskResult objects.""" - task_results = [] - - for result in results: - if isinstance(result, TaskResult): - task_results.append(result) - elif isinstance(result, Exception): - task_results.append(TaskResult( - task_id=f"error_{int(time.time())}", - success=False, - error=str(result) - )) - else: - task_results.append(TaskResult( - task_id=f"success_{int(time.time())}", - success=True, - result=result - )) - - return task_results - - def get_performance_report(self) -> Dict[str, Any]: - """Get comprehensive performance report.""" - rate_limiter_stats = self.rate_limiter.get_stats() - memory_usage = psutil.virtual_memory().percent - cpu_usage = psutil.cpu_percent() - - return { - "dispatcher_stats": { - "total_requests": self.stats.total_requests, - "successful_requests": self.stats.successful_requests, - "failed_requests": self.stats.failed_requests, - "rate_limited_requests": self.stats.rate_limited_requests, - "success_rate": self.stats.successful_requests / max(1, self.stats.total_requests), - "avg_response_time": self.stats.avg_response_time, - "current_active": self.stats.current_active - }, - "resource_usage": { - "memory_percent": memory_usage, - "cpu_percent": cpu_usage, - "peak_memory_usage_bytes": self.stats.peak_memory_usage - }, - "concurrency": { - "max_concurrent": self.max_concurrent, - "current_max_concurrent": self.current_max_concurrent, - "adaptation_count": len(self.adaptation_history) - }, - "rate_limiting": rate_limiter_stats, - "performance_score": self._calculate_performance_score() - } - - -# Factory functions -def create_dispatcher( - dispatcher_type: str = "memory_adaptive", - max_concurrent: int = 10, - **kwargs -) -> BaseDispatcher: - """ - Factory function to create dispatchers. - - Args: - dispatcher_type: Type of dispatcher ("semaphore", "memory_adaptive") - max_concurrent: Maximum concurrent operations - **kwargs: Additional configuration - - Returns: - Configured dispatcher instance - """ - dispatchers = { - "semaphore": SemaphoreDispatcher, - "memory_adaptive": MemoryAdaptiveDispatcher - } - - if dispatcher_type not in dispatchers: - raise ValueError(f"Unknown dispatcher type: {dispatcher_type}. Available: {list(dispatchers.keys())}") - - dispatcher_class = dispatchers[dispatcher_type] - return dispatcher_class(max_concurrent=max_concurrent, **kwargs) - - -def create_rate_limiter( - max_requests: int = 100, - time_window: float = 60.0, - per_domain: bool = True -) -> RateLimiter: - """Create a rate limiter with common settings.""" - return RateLimiter( - max_requests=max_requests, - time_window=time_window, - per_domain=per_domain - ) diff --git a/apps/backend/app/services/enhanced_scraping.py b/apps/backend/app/services/enhanced_scraping.py deleted file mode 100644 index 02a6ea9..0000000 --- a/apps/backend/app/services/enhanced_scraping.py +++ /dev/null @@ -1,643 +0,0 @@ -""" -Enhanced content scraping service with crawl4ai-inspired capabilities. - -This service integrates all the sophisticated features: -- Advanced extraction strategies -- Content filtering -- Enhanced markdown generation -- Adaptive crawling -- Virtual scrolling -- Link analysis -""" - -import asyncio -import json -import time -from typing import Dict, List, Optional, Any, Tuple -from urllib.parse import urljoin, urlparse - -import structlog -from bs4 import BeautifulSoup - -from app.config import get_settings -from app.models.responses import ScrapedContent, ContentMetadata -from app.models.requests import ( - ScrapingConfig, ExtractionStrategyConfig, ContentFilterConfig, - MarkdownConfig, AdaptiveCrawlConfig, VirtualScrollConfig, LinkAnalysisConfig -) -from app.services.scraping import ContentScrapingService -from app.services.extraction_strategies import create_extraction_strategy -from app.services.content_filters import create_content_filter -from app.services.markdown_generation import DefaultMarkdownGenerator -from app.services.adaptive_crawling import create_adaptive_crawler -from app.services.virtual_scrolling import PuppeteerVirtualScroller -from app.services.link_analysis import LinkAnalyzer, LinkPreviewConfig -from app.services.chunking_strategies import create_chunking_strategy -from app.services.table_extraction import create_table_extraction_strategy -from app.services.browser_config import BrowserConfig, create_browser_config_from_env -from app.services.dispatcher import create_dispatcher -from app.services.url_seeder import URLSeeder, SeedingConfig -from app.utils.text_processing import sanitize_text, detect_language, calculate_text_quality - -logger = structlog.get_logger(__name__) -settings = get_settings() - - -class EnhancedScrapingService(ContentScrapingService): - """Enhanced scraping service with crawl4ai-inspired features.""" - - def __init__(self): - """Initialize enhanced scraping service.""" - super().__init__() - - # Initialize advanced components - self.markdown_generator = None - self.adaptive_crawler = None - self.virtual_scroller = None - self.link_analyzer = None - self.url_seeder = None - self.dispatcher = None - self.browser_config = None - - # Initialize dispatcher for concurrent operations - self.dispatcher = create_dispatcher( - dispatcher_type="memory_adaptive", - max_concurrent=settings.scraping_max_concurrent - ) - - async def scrape_urls_enhanced( - self, - urls: List[str], - config: Optional[ScrapingConfig] = None - ) -> List[ScrapedContent]: - """ - Enhanced URL scraping with all advanced features. - - Args: - urls: List of URLs to scrape - config: Enhanced scraping configuration - - Returns: - List of enhanced ScrapedContent objects - """ - if not config: - # Use basic scraping if no advanced config provided - return await super().scrape_urls(urls, config) - - logger.info( - "enhanced_scraping_started", - urls=len(urls), - extraction_strategy=getattr(config, 'extraction_strategy', 'none'), - content_filter=getattr(config, 'content_filter', 'none') - ) - - # Check if adaptive crawling is enabled - if getattr(config, 'adaptive_crawling', False) and len(urls) == 1: - return await self._adaptive_crawl_single_url(urls[0], config) - - # Process URLs with standard enhanced processing - scraped_contents = [] - - for url in urls: - try: - scraped_content = await self._scrape_single_url_enhanced(url, config) - scraped_contents.append(scraped_content) - except Exception as e: - logger.error("enhanced_scraping_failed", url=url, error=str(e)) - # Create error result - scraped_contents.append( - ScrapedContent( - url=url, - title=None, - text="", - extraction_success=False, - extraction_time_ms=0, - word_count=0, - metadata=ContentMetadata(), - error_message=str(e), - content_quality_score=0.0 - ) - ) - - return scraped_contents - - async def _scrape_single_url_enhanced( - self, - url: str, - config: ScrapingConfig - ) -> ScrapedContent: - """Scrape single URL with all enhancements applied.""" - start_time = time.time() - - # Step 1: Basic content extraction - if getattr(config, 'virtual_scrolling', False): - # Use virtual scrolling for infinite pages - basic_result = await self._scrape_with_virtual_scrolling(url, config) - else: - # Standard scraping - basic_result = await super()._scrape_single_url(url, config) - - if not basic_result.extraction_success: - return basic_result - - # Step 2: Apply extraction strategy - extracted_content = await self._apply_extraction_strategy( - url, basic_result.html or basic_result.text, config - ) - - # Step 3: Apply content filtering - filtered_content = await self._apply_content_filtering( - basic_result.html or basic_result.text, config - ) - - # Step 4: Generate enhanced markdown - markdown_result = await self._generate_enhanced_markdown( - filtered_content or basic_result.html or basic_result.text, url, config - ) - - # Step 5: Perform link analysis - link_analysis_result = await self._analyze_links( - basic_result.html or basic_result.text, url, config - ) - - # Step 6: Combine all results into enhanced ScrapedContent - enhanced_result = await self._combine_enhanced_results( - basic_result, extracted_content, filtered_content, - markdown_result, link_analysis_result, config - ) - - processing_time = time.time() - start_time - enhanced_result.extraction_time_ms = int(processing_time * 1000) - - logger.info( - "enhanced_scraping_completed", - url=url, - processing_time=processing_time, - extraction_strategy=getattr(config, 'extraction_strategy', 'none'), - content_filter=getattr(config, 'content_filter', 'none') - ) - - return enhanced_result - - async def _scrape_with_virtual_scrolling( - self, - url: str, - config: ScrapingConfig - ) -> ScrapedContent: - """Scrape URL with virtual scrolling for infinite pages.""" - try: - # Parse virtual scroll configuration - virtual_config = getattr(config, 'virtual_scroll_config', {}) - from app.services.virtual_scrolling import VirtualScrollConfig - - scroll_config = VirtualScrollConfig(**virtual_config) - - # Initialize virtual scroller - if not self.virtual_scroller: - self.virtual_scroller = PuppeteerVirtualScroller( - str(settings.puppeteer_service_url) - ) - - # Perform virtual scrolling - scroll_result = await self.virtual_scroller.scroll_and_extract( - url=url, - config=scroll_config, - headers=getattr(config, 'headers', None) - ) - - if scroll_result.success: - # Convert virtual scroll result to ScrapedContent - soup = BeautifulSoup(scroll_result.final_content, 'lxml') - - # Extract metadata - metadata = await super().extract_metadata(soup, url) - - # Extract images and links - images = super()._extract_images(soup, url) if getattr(config, 'extract_images', True) else [] - links = super()._extract_links(soup, url) if getattr(config, 'extract_links', True) else [] - - # Get clean text - for element in soup(['script', 'style', 'noscript']): - element.decompose() - text = sanitize_text(soup.get_text()) - - return ScrapedContent( - url=url, - title=soup.find('title').get_text() if soup.find('title') else metadata.title, - text=text, - html=scroll_result.final_content, - images=images, - links=links, - metadata=metadata, - extraction_success=True, - extraction_time_ms=int(scroll_result.performance_metrics.get('total_time', 0) * 1000), - word_count=len(text.split()) if text else 0, - language_detected=detect_language(text), - content_quality_score=calculate_text_quality(text), - # Add virtual scrolling metadata - **{ - 'virtual_scrolling_metadata': scroll_result.scroll_metadata, - 'performance_metrics': scroll_result.performance_metrics - } - ) - else: - # Fallback to regular scraping - return await super()._scrape_single_url(url, config) - - except Exception as e: - logger.error("virtual_scrolling_failed", url=url, error=str(e)) - # Fallback to regular scraping - return await super()._scrape_single_url(url, config) - - async def _apply_extraction_strategy( - self, - url: str, - content: str, - config: ScrapingConfig - ) -> Optional[List[Dict[str, Any]]]: - """Apply selected extraction strategy to content.""" - strategy_type = getattr(config, 'extraction_strategy', 'none') - - if strategy_type == 'none': - return None - - try: - # Parse extraction configuration - extraction_config = getattr(config, 'extraction_config', {}) - - # Create extraction strategy - strategy = create_extraction_strategy(strategy_type, extraction_config) - - # Apply extraction - extracted_blocks = await strategy.extract(url, content) - - logger.debug( - "extraction_strategy_applied", - strategy=strategy_type, - blocks_extracted=len(extracted_blocks) - ) - - return extracted_blocks - - except Exception as e: - logger.error( - "extraction_strategy_failed", - strategy=strategy_type, - error=str(e) - ) - return None - - async def _apply_content_filtering( - self, - content: str, - config: ScrapingConfig - ) -> Optional[str]: - """Apply content filtering to HTML content.""" - filter_type = getattr(config, 'content_filter', 'none') - - if filter_type == 'none': - return None - - try: - # Parse filter configuration - filter_config = getattr(config, 'content_filter_config', {}) - - # Create content filter - content_filter = create_content_filter(filter_type, filter_config) - - # Apply filtering - filter_result = await content_filter.filter(content) - - logger.debug( - "content_filter_applied", - filter_type=filter_type, - original_length=filter_result.original_length, - filtered_length=filter_result.filtered_length, - relevance_score=filter_result.relevance_score - ) - - return filter_result.filtered_content - - except Exception as e: - logger.error( - "content_filter_failed", - filter_type=filter_type, - error=str(e) - ) - return None - - async def _generate_enhanced_markdown( - self, - content: str, - url: str, - config: ScrapingConfig - ) -> Optional[Any]: - """Generate enhanced markdown with citations.""" - if not getattr(config, 'markdown_generation', False): - return None - - try: - # Parse markdown configuration - markdown_config = getattr(config, 'markdown_config', {}) - - # Create content filter for fit markdown if specified - content_filter = None - if 'content_filter' in markdown_config: - filter_config = markdown_config['content_filter'] - content_filter = create_content_filter( - filter_config.get('filter_type', 'none'), - filter_config - ) - - # Initialize markdown generator - if not self.markdown_generator: - self.markdown_generator = DefaultMarkdownGenerator( - content_filter=content_filter, - options=markdown_config - ) - - # Generate markdown - markdown_result = await self.markdown_generator.generate_markdown( - input_html=content, - base_url=url, - citations=markdown_config.get('citations', True) - ) - - logger.debug( - "enhanced_markdown_generated", - raw_length=len(markdown_result.raw_markdown), - fit_length=len(markdown_result.fit_markdown) if markdown_result.fit_markdown else 0, - citations_count=len(markdown_result.citation_map) if markdown_result.citation_map else 0 - ) - - return markdown_result - - except Exception as e: - logger.error("enhanced_markdown_failed", error=str(e)) - return None - - async def _analyze_links( - self, - content: str, - url: str, - config: ScrapingConfig - ) -> Optional[Any]: - """Perform intelligent link analysis.""" - if not getattr(config, 'link_analysis', False): - return None - - try: - # Parse link analysis configuration - link_config = getattr(config, 'link_analysis_config', {}) - - # Create link analysis configuration - from app.services.link_analysis import LinkPreviewConfig - preview_config = LinkPreviewConfig(**link_config) - - # Initialize link analyzer - if not self.link_analyzer: - from app.services.link_analysis import LinkAnalyzer - self.link_analyzer = LinkAnalyzer(preview_config) - - # Perform link analysis - analysis_result = await self.link_analyzer.analyze_page_links( - html_content=content, - base_url=url - ) - - logger.debug( - "link_analysis_completed", - total_links=analysis_result.analysis_metadata.get('total_links_found', 0), - above_threshold=analysis_result.analysis_metadata.get('links_above_threshold', 0) - ) - - return analysis_result - - except Exception as e: - logger.error("link_analysis_failed", error=str(e)) - return None - - async def _adaptive_crawl_single_url( - self, - start_url: str, - config: ScrapingConfig - ) -> List[ScrapedContent]: - """Perform adaptive crawling starting from a single URL.""" - try: - # Parse adaptive crawling configuration - adaptive_config = getattr(config, 'adaptive_config', {}) - from app.services.adaptive_crawling import AdaptiveConfig - - crawl_config = AdaptiveConfig(**adaptive_config) - - # Create adaptive crawler - if not self.adaptive_crawler: - from app.services.adaptive_crawling import AdaptiveCrawler - self.adaptive_crawler = AdaptiveCrawler( - scraping_service=self, - config=crawl_config - ) - - # Perform adaptive crawling - crawl_result = await self.adaptive_crawler.adaptive_crawl( - start_url=start_url, - query=getattr(config, 'link_score_query', '') or '', - max_links_to_follow=crawl_config.max_pages - ) - - logger.info( - "adaptive_crawling_completed", - urls_crawled=crawl_result['crawl_state']['urls_crawled'], - confidence=crawl_result['crawl_state']['confidence'] - ) - - return crawl_result['results'] - - except Exception as e: - logger.error("adaptive_crawling_failed", error=str(e)) - # Fallback to single URL scraping - basic_result = await super()._scrape_single_url(start_url, config) - return [basic_result] - - async def _combine_enhanced_results( - self, - basic_result: ScrapedContent, - extracted_content: Optional[List[Dict[str, Any]]], - filtered_content: Optional[str], - markdown_result: Optional[Any], - link_analysis_result: Optional[Any], - config: ScrapingConfig - ) -> ScrapedContent: - """Combine all enhancement results into final ScrapedContent.""" - - # Start with basic result - enhanced_data = { - 'url': basic_result.url, - 'title': basic_result.title, - 'text': basic_result.text, - 'html': basic_result.html, - 'images': basic_result.images, - 'links': basic_result.links, - 'metadata': basic_result.metadata, - 'extraction_success': basic_result.extraction_success, - 'extraction_time_ms': basic_result.extraction_time_ms, - 'word_count': basic_result.word_count, - 'language_detected': basic_result.language_detected, - 'content_quality_score': basic_result.content_quality_score, - 'error_message': basic_result.error_message - } - - # Add extraction results - if extracted_content: - enhanced_data['extracted_content'] = extracted_content - enhanced_data['extraction_strategy'] = getattr(config, 'extraction_strategy', 'none') - - # Add filtered content - if filtered_content: - enhanced_data['filtered_html'] = filtered_content - enhanced_data['content_filter'] = getattr(config, 'content_filter', 'none') - - # Update text with filtered content if using markdown output - if getattr(config, 'response_format', 'json') == 'markdown': - soup = BeautifulSoup(filtered_content, 'lxml') - enhanced_data['text'] = sanitize_text(soup.get_text()) - - # Add markdown results - if markdown_result: - enhanced_data['markdown'] = { - 'raw_markdown': markdown_result.raw_markdown, - 'fit_markdown': markdown_result.fit_markdown, - 'references_markdown': markdown_result.references_markdown, - 'citation_map': markdown_result.citation_map, - 'link_analysis': markdown_result.link_analysis, - 'generation_metadata': markdown_result.generation_metadata - } - - # Update text with markdown if requested - if getattr(config, 'response_format', 'json') == 'markdown': - enhanced_data['text'] = markdown_result.fit_markdown or markdown_result.raw_markdown - - # Add link analysis results - if link_analysis_result: - enhanced_data['link_analysis'] = { - 'top_links': [ - { - 'url': link.url, - 'text': link.text, - 'title': link.title, - 'domain': link.domain, - 'overall_score': link.overall_score, - 'relevance_score': link.relevance_score, - 'authority_score': link.authority_score, - 'quality_score': link.quality_score, - 'priority_rank': link.priority_rank - } - for link in link_analysis_result.top_links[:20] # Top 20 links - ], - 'domain_statistics': link_analysis_result.domain_statistics, - 'quality_distribution': link_analysis_result.quality_distribution, - 'analysis_metadata': link_analysis_result.analysis_metadata - } - - return ScrapedContent(**enhanced_data) - - async def extract_tables( - self, - html_content: str, - base_url: str = "", - strategy: str = "default", - config: Optional[Dict[str, Any]] = None - ) -> List[Dict[str, Any]]: - """Extract tables from HTML content.""" - try: - extractor = create_table_extraction_strategy(strategy, config or {}) - return extractor.extract_tables(html_content, base_url) - except Exception as e: - logger.error("table_extraction_failed", error=str(e)) - return [] - - async def chunk_content( - self, - text: str, - strategy: str = "paragraph", - config: Optional[Dict[str, Any]] = None - ) -> List[str]: - """Chunk text content using specified strategy.""" - try: - chunker = create_chunking_strategy(strategy, config or {}) - return chunker.chunk(text) - except Exception as e: - logger.error("content_chunking_failed", error=str(e)) - return [text] # Fallback to original text - - async def discover_urls( - self, - base_url: str, - config: Optional[Dict[str, Any]] = None - ) -> List[Dict[str, Any]]: - """Discover URLs using URL seeder.""" - try: - seeding_config = SeedingConfig(**config) if config else SeedingConfig() - - if not self.url_seeder: - self.url_seeder = URLSeeder(seeding_config) - await self.url_seeder.initialize() - - discovered_urls = await self.url_seeder.discover(base_url) - return [url.to_dict() for url in discovered_urls] - except Exception as e: - logger.error("url_discovery_failed", error=str(e)) - return [] - - async def get_performance_report(self) -> Dict[str, Any]: - """Get comprehensive performance report.""" - report = { - "enhanced_scraping": { - "components_initialized": { - "markdown_generator": self.markdown_generator is not None, - "adaptive_crawler": self.adaptive_crawler is not None, - "virtual_scroller": self.virtual_scroller is not None, - "link_analyzer": self.link_analyzer is not None, - "url_seeder": self.url_seeder is not None, - "dispatcher": self.dispatcher is not None - } - } - } - - # Add dispatcher performance report - if self.dispatcher and hasattr(self.dispatcher, 'get_performance_report'): - report["dispatcher"] = self.dispatcher.get_performance_report() - - return report - - async def cleanup(self): - """Cleanup all resources.""" - try: - # Cleanup dispatcher - if self.dispatcher: - await self.dispatcher.cleanup() - - # Cleanup URL seeder - if self.url_seeder: - await self.url_seeder.close() - - # Call parent cleanup - await super().close() - - except Exception as e: - logger.error("cleanup_failed", error=str(e)) - - -# Singleton instance -_enhanced_scraping_service: Optional[EnhancedScrapingService] = None - - -async def get_enhanced_scraping_service() -> EnhancedScrapingService: - """Get or create enhanced scraping service instance.""" - global _enhanced_scraping_service - - if _enhanced_scraping_service is None: - _enhanced_scraping_service = EnhancedScrapingService() - await _enhanced_scraping_service.initialize() - - return _enhanced_scraping_service diff --git a/apps/backend/app/services/extraction_strategies.py b/apps/backend/app/services/extraction_strategies.py deleted file mode 100644 index 2272526..0000000 --- a/apps/backend/app/services/extraction_strategies.py +++ /dev/null @@ -1,664 +0,0 @@ -""" -Advanced content extraction strategies inspired by crawl4ai. - -This module implements sophisticated extraction strategies for different use cases: -- LLMExtractionStrategy: AI-powered structured data extraction -- CosineStrategy: Semantic similarity clustering for content extraction -- JsonCssExtractionStrategy: Advanced CSS/XPath extraction with schema -- RegexExtractionStrategy: Pattern-based extraction -- NoExtractionStrategy: Simple pass-through strategy -""" - -import asyncio -import json -import re -import math -from abc import ABC, abstractmethod -from typing import Dict, List, Optional, Any, Union, Pattern, Tuple -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass -from urllib.parse import urljoin - -import httpx -import numpy as np -from bs4 import BeautifulSoup -from lxml import html, etree -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.cluster import AgglomerativeClustering -from sklearn.metrics.pairwise import cosine_similarity -import structlog - -from app.utils.text_processing import sanitize_text, clean_tokens -from app.models.responses import ExtractedContent, ExtractionMetadata - -logger = structlog.get_logger(__name__) - - -@dataclass -class TokenUsage: - """Token usage tracking for LLM operations.""" - completion_tokens: int = 0 - prompt_tokens: int = 0 - total_tokens: int = 0 - completion_tokens_details: Optional[dict] = None - prompt_tokens_details: Optional[dict] = None - - -class ExtractionStrategy(ABC): - """Abstract base class for all extraction strategies.""" - - def __init__(self, input_format: str = "markdown", **kwargs): - """ - Initialize the extraction strategy. - - Args: - input_format: Content format to use for extraction. - Options: "markdown" (default), "html", "fit_markdown" - **kwargs: Additional keyword arguments - """ - self.input_format = input_format - self.name = self.__class__.__name__ - self.verbose = kwargs.get("verbose", False) - - @abstractmethod - async def extract(self, url: str, html: str, **kwargs) -> List[Dict[str, Any]]: - """ - Extract meaningful blocks or chunks from the given HTML. - - Args: - url: The URL of the webpage. - html: The HTML content of the webpage. - **kwargs: Additional extraction parameters. - - Returns: - A list of extracted blocks or chunks. - """ - pass - - async def run(self, url: str, sections: List[str], **kwargs) -> List[Dict[str, Any]]: - """ - Process sections of text in parallel. - - Args: - url: The URL of the webpage. - sections: List of sections (strings) to process. - **kwargs: Additional extraction parameters. - - Returns: - A list of processed JSON blocks. - """ - extracted_content = [] - - # Use ThreadPoolExecutor for CPU-bound operations - with ThreadPoolExecutor() as executor: - loop = asyncio.get_event_loop() - futures = [ - loop.run_in_executor(executor, self.extract_sync, url, section, **kwargs) - for section in sections - ] - - for future in asyncio.as_completed(futures): - try: - result = await future - extracted_content.extend(result) - except Exception as e: - logger.error("extraction_section_failed", error=str(e)) - - return extracted_content - - def extract_sync(self, url: str, html: str, **kwargs) -> List[Dict[str, Any]]: - """Synchronous wrapper for extract method.""" - # This will be overridden by async strategies - return [] - - -class NoExtractionStrategy(ExtractionStrategy): - """A strategy that returns the entire HTML as a single block.""" - - async def extract(self, url: str, html: str, **kwargs) -> List[Dict[str, Any]]: - """Extract the entire HTML as a single block.""" - return [{"index": 0, "content": html}] - - async def run(self, url: str, sections: List[str], **kwargs) -> List[Dict[str, Any]]: - """Process sections without any extraction.""" - return [ - {"index": i, "tags": [], "content": section} - for i, section in enumerate(sections) - ] - - -class CosineStrategy(ExtractionStrategy): - """ - Extract meaningful blocks using cosine similarity clustering. - - This strategy: - 1. Pre-filters documents using embeddings and semantic_filter - 2. Performs clustering using cosine similarity - 3. Organizes texts by their cluster labels, retaining order - 4. Filters clusters by word count - 5. Extracts meaningful blocks from filtered clusters - """ - - def __init__( - self, - semantic_filter: Optional[str] = None, - word_count_threshold: int = 10, - max_dist: float = 0.2, - linkage_method: str = "ward", - top_k: int = 3, - model_name: str = "sentence-transformers/all-MiniLM-L6-v2", - sim_threshold: float = 0.3, - **kwargs - ): - super().__init__(**kwargs) - self.semantic_filter = semantic_filter - self.word_count_threshold = word_count_threshold - self.max_dist = max_dist - self.linkage_method = linkage_method - self.top_k = top_k - self.model_name = model_name - self.sim_threshold = sim_threshold - - async def extract(self, url: str, html: str, **kwargs) -> List[Dict[str, Any]]: - """Extract content using cosine similarity clustering.""" - try: - # Parse HTML and extract text blocks - soup = BeautifulSoup(html, 'lxml') - text_blocks = self._extract_text_blocks(soup) - - if not text_blocks: - return [] - - # Apply semantic filtering if specified - if self.semantic_filter: - text_blocks = self._apply_semantic_filter(text_blocks) - - # Perform TF-IDF vectorization - vectorizer = TfidfVectorizer( - max_features=1000, - stop_words='english', - ngram_range=(1, 2) - ) - - try: - tfidf_matrix = vectorizer.fit_transform(text_blocks) - except ValueError: - # Handle case where all documents are empty - return [{"index": i, "content": block} for i, block in enumerate(text_blocks)] - - # Perform clustering - if len(text_blocks) > 1: - clustering = AgglomerativeClustering( - n_clusters=None, - distance_threshold=self.max_dist, - linkage=self.linkage_method - ) - cluster_labels = clustering.fit_predict(tfidf_matrix.toarray()) - else: - cluster_labels = [0] - - # Group by clusters and filter by word count - clusters = {} - for idx, (text, label) in enumerate(zip(text_blocks, cluster_labels)): - if label not in clusters: - clusters[label] = [] - clusters[label].append({ - "index": idx, - "content": text, - "word_count": len(text.split()) - }) - - # Filter clusters by word count and select top-k - filtered_clusters = [] - for label, blocks in clusters.items(): - total_words = sum(block["word_count"] for block in blocks) - if total_words >= self.word_count_threshold: - # Combine blocks in cluster - combined_content = "\n\n".join(block["content"] for block in blocks) - filtered_clusters.append({ - "cluster": label, - "content": combined_content, - "word_count": total_words, - "blocks": blocks - }) - - # Sort by word count and take top-k - filtered_clusters.sort(key=lambda x: x["word_count"], reverse=True) - top_clusters = filtered_clusters[:self.top_k] - - # Return extracted content - results = [] - for i, cluster in enumerate(top_clusters): - results.append({ - "index": i, - "content": cluster["content"], - "cluster": cluster["cluster"], - "word_count": cluster["word_count"], - "blocks_count": len(cluster["blocks"]) - }) - - return results - - except Exception as e: - logger.error("cosine_strategy_extraction_failed", url=url, error=str(e)) - return [] - - def _extract_text_blocks(self, soup: BeautifulSoup) -> List[str]: - """Extract text blocks from HTML.""" - # Remove script and style elements - for element in soup(['script', 'style', 'noscript']): - element.decompose() - - # Extract text from meaningful elements - text_blocks = [] - for element in soup.find_all(['p', 'div', 'article', 'section', 'li', 'td', 'th']): - text = sanitize_text(element.get_text()) - if text and len(text.split()) >= 3: # Minimum 3 words - text_blocks.append(text) - - return text_blocks - - def _apply_semantic_filter(self, text_blocks: List[str]) -> List[str]: - """Apply semantic filtering based on keyword similarity.""" - if not self.semantic_filter: - return text_blocks - - filter_keywords = set(self.semantic_filter.lower().split()) - filtered_blocks = [] - - for block in text_blocks: - block_words = set(block.lower().split()) - # Calculate Jaccard similarity - intersection = len(filter_keywords.intersection(block_words)) - union = len(filter_keywords.union(block_words)) - similarity = intersection / union if union > 0 else 0 - - if similarity >= self.sim_threshold: - filtered_blocks.append(block) - - return filtered_blocks if filtered_blocks else text_blocks - - -class JsonCssExtractionStrategy(ExtractionStrategy): - """ - Advanced CSS/XPath extraction with schema-based data extraction. - - Supports extracting structured data using CSS selectors and XPath expressions - with a defined schema for consistent output formatting. - """ - - def __init__(self, schema: Dict[str, Any], **kwargs): - """ - Initialize with extraction schema. - - Args: - schema: JSON schema defining extraction rules - Example schema: - { - "name": "Product Extractor", - "baseSelector": ".product", - "fields": [ - {"name": "title", "selector": "h2", "type": "text"}, - {"name": "price", "selector": ".price", "type": "text"}, - {"name": "image", "selector": "img", "type": "attribute", "attribute": "src"} - ] - } - """ - super().__init__(**kwargs) - self.schema = schema - self.validate_schema() - - def validate_schema(self): - """Validate the extraction schema.""" - required_fields = ["name", "baseSelector", "fields"] - for field in required_fields: - if field not in self.schema: - raise ValueError(f"Schema missing required field: {field}") - - if not isinstance(self.schema["fields"], list): - raise ValueError("Schema 'fields' must be a list") - - for field in self.schema["fields"]: - if not isinstance(field, dict) or "name" not in field or "selector" not in field: - raise ValueError("Each field must have 'name' and 'selector'") - - async def extract(self, url: str, html: str, **kwargs) -> List[Dict[str, Any]]: - """Extract structured data using the schema.""" - try: - soup = BeautifulSoup(html, 'lxml') - base_elements = soup.select(self.schema["baseSelector"]) - - if not base_elements: - if self.verbose: - logger.warning("no_base_elements_found", - url=url, - selector=self.schema["baseSelector"]) - return [] - - results = [] - for idx, base_element in enumerate(base_elements): - extracted_item = {"_index": idx} - - for field in self.schema["fields"]: - field_name = field["name"] - selector = field["selector"] - field_type = field.get("type", "text") - attribute = field.get("attribute", None) - - # Find element within base element - target_element = base_element.select_one(selector) - - if target_element: - if field_type == "text": - extracted_item[field_name] = sanitize_text(target_element.get_text()) - elif field_type == "attribute" and attribute: - attr_value = target_element.get(attribute, "") - # Make URLs absolute if needed - if attribute in ["href", "src"] and attr_value: - extracted_item[field_name] = urljoin(url, attr_value) - else: - extracted_item[field_name] = attr_value - elif field_type == "html": - extracted_item[field_name] = str(target_element) - else: - extracted_item[field_name] = sanitize_text(target_element.get_text()) - else: - extracted_item[field_name] = "" - - results.append(extracted_item) - - return results - - except Exception as e: - logger.error("json_css_extraction_failed", url=url, error=str(e)) - return [] - - -class RegexExtractionStrategy(ExtractionStrategy): - """ - Pattern-based extraction using regular expressions. - - Supports extracting data using multiple regex patterns with named groups - for structured output. - """ - - def __init__(self, patterns: Dict[str, Union[str, Pattern]], **kwargs): - """ - Initialize with regex patterns. - - Args: - patterns: Dictionary of pattern names to regex patterns - Example: - { - "emails": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "phones": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b", - "prices": r"\$[\d,]+\.?\d*" - } - """ - super().__init__(**kwargs) - self.patterns = {} - - # Compile patterns - for name, pattern in patterns.items(): - if isinstance(pattern, str): - self.patterns[name] = re.compile(pattern, re.IGNORECASE | re.MULTILINE) - else: - self.patterns[name] = pattern - - async def extract(self, url: str, html: str, **kwargs) -> List[Dict[str, Any]]: - """Extract content using regex patterns.""" - try: - # Parse HTML to get clean text - soup = BeautifulSoup(html, 'lxml') - - # Remove script and style elements - for element in soup(['script', 'style', 'noscript']): - element.decompose() - - text_content = soup.get_text() - results = [] - - # Apply each pattern - for pattern_name, pattern in self.patterns.items(): - matches = pattern.findall(text_content) - - if matches: - for i, match in enumerate(matches): - if isinstance(match, tuple): - # Named groups - match_dict = { - "pattern": pattern_name, - "index": i, - "match": match[0] if match else "", - "groups": list(match) - } - else: - # Simple match - match_dict = { - "pattern": pattern_name, - "index": i, - "match": match - } - - results.append(match_dict) - - # If no matches found, return the text in chunks - if not results: - text_chunks = text_content.split('\n\n') - for i, chunk in enumerate(text_chunks): - chunk = chunk.strip() - if chunk and len(chunk.split()) >= 5: - results.append({ - "pattern": "text_chunk", - "index": i, - "content": sanitize_text(chunk) - }) - - return results - - except Exception as e: - logger.error("regex_extraction_failed", url=url, error=str(e)) - return [] - - -class LLMExtractionStrategy(ExtractionStrategy): - """ - AI-powered structured data extraction using language models. - - This strategy uses LLMs to extract structured data based on: - - Custom schemas - - Natural language instructions - - Contextual understanding - """ - - def __init__( - self, - llm_config: Optional[Dict[str, Any]] = None, - schema: Optional[Dict[str, Any]] = None, - extraction_type: str = "schema", - instruction: Optional[str] = None, - **kwargs - ): - """ - Initialize LLM extraction strategy. - - Args: - llm_config: Configuration for LLM provider (API keys, model, etc.) - schema: JSON schema for structured extraction - extraction_type: Type of extraction ("schema", "instruction", "blocks") - instruction: Natural language instruction for extraction - """ - super().__init__(**kwargs) - self.llm_config = llm_config or {} - self.schema = schema - self.extraction_type = extraction_type - self.instruction = instruction - self.token_usage = TokenUsage() - - async def extract(self, url: str, html: str, **kwargs) -> List[Dict[str, Any]]: - """Extract content using LLM.""" - try: - # Parse and clean HTML - soup = BeautifulSoup(html, 'lxml') - - # Remove script and style elements - for element in soup(['script', 'style', 'noscript']): - element.decompose() - - # Get clean text content - text_content = sanitize_text(soup.get_text()) - - # Truncate if too long (to fit LLM context) - max_chars = kwargs.get('max_chars', 10000) - if len(text_content) > max_chars: - text_content = text_content[:max_chars] + "..." - - # Prepare prompt based on extraction type - if self.extraction_type == "schema" and self.schema: - prompt = self._build_schema_prompt(text_content, url) - elif self.extraction_type == "instruction" and self.instruction: - prompt = self._build_instruction_prompt(text_content, url) - else: - prompt = self._build_blocks_prompt(text_content, url) - - # Make LLM API call (mock implementation for now) - # In production, this would call actual LLM APIs like OpenAI, etc. - result = await self._call_llm_api(prompt, **kwargs) - - return result - - except Exception as e: - logger.error("llm_extraction_failed", url=url, error=str(e)) - return [] - - def _build_schema_prompt(self, content: str, url: str) -> str: - """Build prompt for schema-based extraction.""" - schema_str = json.dumps(self.schema, indent=2) - - prompt = f""" -Extract structured data from the following content according to this JSON schema: - -Schema: -{schema_str} - -Content from {url}: -{content} - -Please return the extracted data as a JSON array of objects matching the schema. -If no data matches the schema, return an empty array. -""" - return prompt - - def _build_instruction_prompt(self, content: str, url: str) -> str: - """Build prompt for instruction-based extraction.""" - prompt = f""" -{self.instruction} - -Content from {url}: -{content} - -Please extract the requested information and return it as structured JSON. -""" - return prompt - - def _build_blocks_prompt(self, content: str, url: str) -> str: - """Build prompt for general block extraction.""" - prompt = f""" -Analyze the following content and extract the most important and meaningful blocks of information. -Focus on the main content, key facts, and important details. - -Content from {url}: -{content} - -Please return the extracted blocks as a JSON array with each block containing: -- "index": sequential number -- "content": the extracted text block -- "importance": relevance score (1-10) -- "category": type of content (article, list, table, etc.) -""" - return prompt - - async def _call_llm_api(self, prompt: str, **kwargs) -> List[Dict[str, Any]]: - """ - Call LLM API for extraction. - - Note: This is a mock implementation. In production, you would integrate - with actual LLM providers like OpenAI, Anthropic, etc. - """ - # Mock response - in production, replace with actual LLM API calls - await asyncio.sleep(0.1) # Simulate API call delay - - # For now, return a simple structured response - # This would be replaced with actual LLM API integration - mock_response = [ - { - "index": 0, - "content": "Mock extracted content - replace with actual LLM integration", - "confidence": 0.8, - "source": "llm_extraction" - } - ] - - # Update token usage (mock) - self.token_usage.prompt_tokens += len(prompt.split()) - self.token_usage.completion_tokens += 50 - self.token_usage.total_tokens = self.token_usage.prompt_tokens + self.token_usage.completion_tokens - - return mock_response - - -# Factory function for creating extraction strategies -def create_extraction_strategy( - strategy_type: str, - config: Dict[str, Any] -) -> ExtractionStrategy: - """ - Factory function to create extraction strategies. - - Args: - strategy_type: Type of strategy ("cosine", "json_css", "regex", "llm", "none") - config: Configuration dictionary for the strategy - - Returns: - Configured extraction strategy instance - """ - strategies = { - "cosine": CosineStrategy, - "json_css": JsonCssExtractionStrategy, - "regex": RegexExtractionStrategy, - "llm": LLMExtractionStrategy, - "none": NoExtractionStrategy - } - - if strategy_type not in strategies: - raise ValueError(f"Unknown strategy type: {strategy_type}. Available: {list(strategies.keys())}") - - strategy_class = strategies[strategy_type] - return strategy_class(**config) - - -# Convenience functions for common extraction patterns -async def extract_with_schema(html: str, url: str, schema: Dict[str, Any]) -> List[Dict[str, Any]]: - """Extract data using JSON CSS schema.""" - strategy = JsonCssExtractionStrategy(schema=schema) - return await strategy.extract(url, html) - - -async def extract_with_patterns(html: str, url: str, patterns: Dict[str, str]) -> List[Dict[str, Any]]: - """Extract data using regex patterns.""" - strategy = RegexExtractionStrategy(patterns=patterns) - return await strategy.extract(url, html) - - -async def extract_with_clustering( - html: str, - url: str, - semantic_filter: Optional[str] = None, - top_k: int = 3 -) -> List[Dict[str, Any]]: - """Extract content using cosine similarity clustering.""" - strategy = CosineStrategy( - semantic_filter=semantic_filter, - top_k=top_k - ) - return await strategy.extract(url, html) diff --git a/apps/backend/app/services/html_converter.py b/apps/backend/app/services/html_converter.py deleted file mode 100644 index dbdf009..0000000 --- a/apps/backend/app/services/html_converter.py +++ /dev/null @@ -1,631 +0,0 @@ -""" -Advanced HTML to text/markdown conversion system with intelligent parsing. - -This module provides sophisticated HTML conversion: -- HTML to clean text conversion -- HTML to structured markdown -- Intelligent content extraction -- Link and image handling -- Table structure preservation -""" - -import re -from typing import Dict, List, Optional, Any, Callable -from dataclasses import dataclass -from html.parser import HTMLParser -from urllib.parse import urljoin, urlparse - -import structlog -from bs4 import BeautifulSoup, Comment - -logger = structlog.get_logger(__name__) - - -@dataclass -class ConversionConfig: - """Configuration for HTML conversion.""" - body_width: int = 78 - skip_internal_links: bool = False - inline_links: bool = True - links_each_paragraph: bool = False - images_to_alt: bool = True - images_with_size: bool = False - ignore_images: bool = False - ignore_links: bool = False - ignore_emphasis: bool = False - ignore_tables: bool = False - escape_special: bool = True - mark_code: bool = True - wrap_links: bool = True - wrap_list_items: bool = True - - # Advanced options - preserve_whitespace: bool = False - decode_errors: str = 'ignore' - baseurl: str = "" - open_quote: str = '"' - close_quote: str = '"' - - # Element handling - emphasis_mark: str = "*" - strong_mark: str = "**" - list_marker: str = "- " - code_mark: str = "`" - - -class HTMLToTextConverter: - """ - Advanced HTML to text converter with intelligent parsing. - - Provides clean text extraction with optional markdown formatting. - """ - - def __init__(self, config: ConversionConfig = None): - """Initialize converter with configuration.""" - self.config = config or ConversionConfig() - - # Conversion state - self.out = [] - self.quiet = 0 - self.p_p = 0 # Number of newlines before current line - self.outcount = 0 - self.start = True - self.space = False - - # Link handling - self.a = [] - self.astack = [] - self.acount = 0 - - # List handling - self.list = [] - self.blockquote = 0 - self.pre = False - - # Table handling - self.table = False - self.td_count = 0 - self.tr_count = 0 - - # Emphasis tracking - self.emphasis = 0 - self.strong = 0 - self.code = False - - # Special characters - self.abbr_data = {} - self.abbr_list = {} - - def convert(self, html: str, baseurl: str = "") -> str: - """ - Convert HTML to clean text. - - Args: - html: HTML content to convert - baseurl: Base URL for resolving relative links - - Returns: - Clean text representation - """ - self.config.baseurl = baseurl or self.config.baseurl - - # Reset state - self._reset_state() - - try: - # Parse HTML with BeautifulSoup for better handling - soup = BeautifulSoup(html, 'lxml') - - # Remove unwanted elements - self._remove_unwanted_elements(soup) - - # Convert to text - self._process_element(soup) - - # Post-process output - return self._finalize_output() - - except Exception as e: - logger.error(f"HTML conversion failed: {str(e)}") - # Fallback to simple text extraction - return self._simple_text_extraction(html) - - def _reset_state(self): - """Reset conversion state.""" - self.out = [] - self.quiet = 0 - self.p_p = 0 - self.outcount = 0 - self.start = True - self.space = False - self.a = [] - self.astack = [] - self.acount = 0 - self.list = [] - self.blockquote = 0 - self.pre = False - self.table = False - self.td_count = 0 - self.tr_count = 0 - self.emphasis = 0 - self.strong = 0 - self.code = False - - def _remove_unwanted_elements(self, soup: BeautifulSoup): - """Remove unwanted HTML elements.""" - # Remove comments - for comment in soup.find_all(string=lambda text: isinstance(text, Comment)): - comment.extract() - - # Remove script and style elements - for script in soup(["script", "style", "noscript", "meta"]): - script.decompose() - - # Remove hidden elements - for element in soup.find_all(style=re.compile(r'display:\s*none|visibility:\s*hidden')): - element.decompose() - - def _process_element(self, element): - """Process HTML element recursively.""" - if hasattr(element, 'name'): - if element.name: - self._handle_tag(element, opening=True) - - # Process children - if hasattr(element, 'children'): - for child in element.children: - if hasattr(child, 'name'): - self._process_element(child) - else: - # Text node - self._handle_text(str(child)) - - if hasattr(element, 'name'): - if element.name: - self._handle_tag(element, opening=False) - - def _handle_tag(self, element, opening: bool = True): - """Handle HTML tag opening/closing.""" - tag = element.name.lower() - - if opening: - self._handle_opening_tag(tag, element) - else: - self._handle_closing_tag(tag, element) - - def _handle_opening_tag(self, tag: str, element): - """Handle opening HTML tags.""" - if tag in ['p', 'div', 'br']: - self._handle_paragraph() - - elif tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: - self._handle_heading(tag, element) - - elif tag == 'a' and not self.config.ignore_links: - self._handle_link_open(element) - - elif tag == 'img' and not self.config.ignore_images: - self._handle_image(element) - - elif tag in ['strong', 'b'] and not self.config.ignore_emphasis: - self._handle_strong_open() - - elif tag in ['em', 'i'] and not self.config.ignore_emphasis: - self._handle_emphasis_open() - - elif tag == 'code' and self.config.mark_code: - self._handle_code_open() - - elif tag in ['ul', 'ol']: - self._handle_list_open(tag, element) - - elif tag == 'li': - self._handle_list_item_open() - - elif tag == 'blockquote': - self._handle_blockquote_open() - - elif tag == 'table' and not self.config.ignore_tables: - self._handle_table_open() - - elif tag == 'tr' and self.table: - self._handle_table_row_open() - - elif tag in ['td', 'th'] and self.table: - self._handle_table_cell_open() - - elif tag == 'pre': - self.pre = True - - def _handle_closing_tag(self, tag: str, element): - """Handle closing HTML tags.""" - if tag == 'a' and not self.config.ignore_links: - self._handle_link_close() - - elif tag in ['strong', 'b'] and not self.config.ignore_emphasis: - self._handle_strong_close() - - elif tag in ['em', 'i'] and not self.config.ignore_emphasis: - self._handle_emphasis_close() - - elif tag == 'code' and self.config.mark_code: - self._handle_code_close() - - elif tag in ['ul', 'ol']: - self._handle_list_close() - - elif tag == 'li': - self._handle_list_item_close() - - elif tag == 'blockquote': - self._handle_blockquote_close() - - elif tag == 'table' and not self.config.ignore_tables: - self._handle_table_close() - - elif tag == 'tr' and self.table: - self._handle_table_row_close() - - elif tag in ['td', 'th'] and self.table: - self._handle_table_cell_close() - - elif tag == 'pre': - self.pre = False - - def _handle_text(self, text: str): - """Handle text content.""" - if self.quiet > 0: - return - - # Preserve whitespace in 
 tags
-        if self.pre:
-            self._output(text)
-            return
-        
-        # Normalize whitespace
-        text = re.sub(r'\s+', ' ', text)
-        
-        if text.strip():
-            if self.space:
-                text = ' ' + text.lstrip()
-            self._output(text)
-            self.space = text.endswith(' ')
-        else:
-            self.space = True
-    
-    def _handle_paragraph(self):
-        """Handle paragraph breaks."""
-        self._p()
-    
-    def _handle_heading(self, tag: str, element):
-        """Handle heading elements."""
-        level = int(tag[1])  # h1 -> 1, h2 -> 2, etc.
-        self._p()
-        
-        # Add markdown-style heading markers
-        if self.config.mark_code:  # Reuse this flag for markdown-style output
-            self._output("#" * level + " ")
-    
-    def _handle_link_open(self, element):
-        """Handle opening anchor tag."""
-        href = element.get('href', '')
-        if href:
-            if self.config.baseurl:
-                href = urljoin(self.config.baseurl, href)
-            
-            self.astack.append((href, self.acount))
-            self.a.append(href)
-            self.acount += 1
-            
-            if not self.config.inline_links:
-                self._output(f"[{self.acount}]")
-    
-    def _handle_link_close(self):
-        """Handle closing anchor tag."""
-        if self.astack:
-            href, count = self.astack.pop()
-            if self.config.inline_links:
-                self._output(f" ({href})")
-    
-    def _handle_image(self, element):
-        """Handle image elements."""
-        alt = element.get('alt', '')
-        src = element.get('src', '')
-        
-        if self.config.images_to_alt and alt:
-            self._output(f"[{alt}]")
-        elif src:
-            if self.config.baseurl:
-                src = urljoin(self.config.baseurl, src)
-            self._output(f"[Image: {src}]")
-    
-    def _handle_strong_open(self):
-        """Handle strong/bold opening."""
-        if self.config.mark_code:
-            self._output(self.config.strong_mark)
-        self.strong += 1
-    
-    def _handle_strong_close(self):
-        """Handle strong/bold closing."""
-        if self.strong > 0:
-            self.strong -= 1
-            if self.config.mark_code:
-                self._output(self.config.strong_mark)
-    
-    def _handle_emphasis_open(self):
-        """Handle emphasis/italic opening."""
-        if self.config.mark_code:
-            self._output(self.config.emphasis_mark)
-        self.emphasis += 1
-    
-    def _handle_emphasis_close(self):
-        """Handle emphasis/italic closing."""
-        if self.emphasis > 0:
-            self.emphasis -= 1
-            if self.config.mark_code:
-                self._output(self.config.emphasis_mark)
-    
-    def _handle_code_open(self):
-        """Handle code opening."""
-        self._output(self.config.code_mark)
-        self.code = True
-    
-    def _handle_code_close(self):
-        """Handle code closing."""
-        if self.code:
-            self._output(self.config.code_mark)
-            self.code = False
-    
-    def _handle_list_open(self, tag: str, element):
-        """Handle list opening."""
-        list_type = 'ordered' if tag == 'ol' else 'unordered'
-        start = int(element.get('start', 1)) if tag == 'ol' else 1
-        self.list.append((list_type, start, 0))
-        self._p()
-    
-    def _handle_list_close(self):
-        """Handle list closing."""
-        if self.list:
-            self.list.pop()
-        self._p()
-    
-    def _handle_list_item_open(self):
-        """Handle list item opening."""
-        if self.list:
-            list_type, start, current = self.list[-1]
-            current += 1
-            self.list[-1] = (list_type, start, current)
-            
-            self._p()
-            
-            if list_type == 'ordered':
-                marker = f"{start + current - 1}. "
-            else:
-                marker = self.config.list_marker
-            
-            # Indent nested lists
-            indent = "  " * (len(self.list) - 1)
-            self._output(indent + marker)
-    
-    def _handle_list_item_close(self):
-        """Handle list item closing."""
-        pass  # Handled by paragraph breaks
-    
-    def _handle_blockquote_open(self):
-        """Handle blockquote opening."""
-        self.blockquote += 1
-        self._p()
-    
-    def _handle_blockquote_close(self):
-        """Handle blockquote closing."""
-        if self.blockquote > 0:
-            self.blockquote -= 1
-        self._p()
-    
-    def _handle_table_open(self):
-        """Handle table opening."""
-        self.table = True
-        self.td_count = 0
-        self.tr_count = 0
-        self._p()
-    
-    def _handle_table_close(self):
-        """Handle table closing."""
-        self.table = False
-        self._p()
-    
-    def _handle_table_row_open(self):
-        """Handle table row opening."""
-        if self.tr_count > 0:
-            self._p()
-        self.tr_count += 1
-        self.td_count = 0
-    
-    def _handle_table_row_close(self):
-        """Handle table row closing."""
-        pass
-    
-    def _handle_table_cell_open(self):
-        """Handle table cell opening."""
-        if self.td_count > 0:
-            self._output(" | ")
-        self.td_count += 1
-    
-    def _handle_table_cell_close(self):
-        """Handle table cell closing."""
-        pass
-    
-    def _output(self, text: str):
-        """Output text with proper formatting."""
-        if text:
-            self.out.append(text)
-            self.outcount += len(text)
-            self.start = False
-    
-    def _p(self):
-        """Add paragraph break."""
-        if not self.start:
-            self.out.append('\n\n')
-            self.start = True
-            self.space = False
-    
-    def _finalize_output(self) -> str:
-        """Finalize and clean up output."""
-        result = ''.join(self.out)
-        
-        # Clean up excessive whitespace
-        result = re.sub(r'\n{3,}', '\n\n', result)  # Max 2 consecutive newlines
-        result = re.sub(r'[ \t]+', ' ', result)      # Multiple spaces to single
-        result = result.strip()
-        
-        # Add link references if not inline
-        if not self.config.inline_links and self.a:
-            result += '\n\n'
-            for i, link in enumerate(self.a, 1):
-                result += f'[{i}]: {link}\n'
-        
-        return result
-    
-    def _simple_text_extraction(self, html: str) -> str:
-        """Fallback simple text extraction."""
-        try:
-            soup = BeautifulSoup(html, 'lxml')
-            return soup.get_text(separator=' ', strip=True)
-        except Exception:
-            # Last resort: remove tags with regex
-            text = re.sub(r'<[^>]+>', '', html)
-            return ' '.join(text.split())
-
-
-class HTMLToMarkdownConverter(HTMLToTextConverter):
-    """
-    HTML to Markdown converter with enhanced formatting.
-    
-    Extends text converter to produce proper markdown output.
-    """
-    
-    def __init__(self, config: ConversionConfig = None):
-        """Initialize markdown converter."""
-        super().__init__(config)
-        
-        # Enable markdown features
-        if self.config:
-            self.config.mark_code = True
-            self.config.inline_links = True
-            self.config.emphasis_mark = "*"
-            self.config.strong_mark = "**"
-            self.config.code_mark = "`"
-    
-    def _handle_heading(self, tag: str, element):
-        """Handle heading with proper markdown formatting."""
-        level = int(tag[1])
-        self._p()
-        self._output("#" * level + " ")
-    
-    def _handle_link_close(self):
-        """Handle link with markdown formatting."""
-        if self.astack:
-            href, count = self.astack.pop()
-            self._output(f"]({href})")
-    
-    def _handle_link_open(self, element):
-        """Handle link opening with markdown formatting."""
-        href = element.get('href', '')
-        if href:
-            if self.config.baseurl:
-                href = urljoin(self.config.baseurl, href)
-            
-            self.astack.append((href, self.acount))
-            self.a.append(href)
-            self.acount += 1
-            self._output("[")
-    
-    def _handle_image(self, element):
-        """Handle image with markdown formatting."""
-        alt = element.get('alt', '')
-        src = element.get('src', '')
-        title = element.get('title', '')
-        
-        if src:
-            if self.config.baseurl:
-                src = urljoin(self.config.baseurl, src)
-            
-            markdown_img = f"![{alt}]({src}"
-            if title:
-                markdown_img += f' "{title}"'
-            markdown_img += ")"
-            
-            self._output(markdown_img)
-    
-    def _handle_blockquote_open(self):
-        """Handle blockquote with markdown formatting."""
-        self.blockquote += 1
-        self._p()
-        self._output("> ")
-    
-    def _handle_table_row_close(self):
-        """Handle table row with markdown formatting."""
-        if self.tr_count == 1:
-            # Add separator row after header
-            self._p()
-            self._output("| " + " | ".join(["---"] * max(1, self.td_count)) + " |")
-
-
-# Factory functions
-def create_html_converter(output_format: str = "text", config: ConversionConfig = None):
-    """
-    Create HTML converter based on output format.
-    
-    Args:
-        output_format: "text" or "markdown"
-        config: Conversion configuration
-        
-    Returns:
-        Appropriate converter instance
-    """
-    if output_format.lower() == "markdown":
-        return HTMLToMarkdownConverter(config)
-    else:
-        return HTMLToTextConverter(config)
-
-
-# Convenience functions
-def html_to_text(html: str, 
-                baseurl: str = "",
-                body_width: int = 78,
-                **kwargs) -> str:
-    """Convert HTML to clean text."""
-    config = ConversionConfig(
-        body_width=body_width,
-        baseurl=baseurl,
-        **kwargs
-    )
-    converter = HTMLToTextConverter(config)
-    return converter.convert(html, baseurl)
-
-
-def html_to_markdown(html: str, 
-                    baseurl: str = "",
-                    **kwargs) -> str:
-    """Convert HTML to markdown."""
-    config = ConversionConfig(
-        baseurl=baseurl,
-        mark_code=True,
-        inline_links=True,
-        **kwargs
-    )
-    converter = HTMLToMarkdownConverter(config)
-    return converter.convert(html, baseurl)
-
-
-def extract_clean_text(html: str, **kwargs) -> str:
-    """Extract clean text with minimal formatting."""
-    config = ConversionConfig(
-        ignore_links=True,
-        ignore_images=True,
-        ignore_emphasis=True,
-        mark_code=False,
-        **kwargs
-    )
-    converter = HTMLToTextConverter(config)
-    return converter.convert(html)
diff --git a/apps/backend/app/services/link_analysis.py b/apps/backend/app/services/link_analysis.py
deleted file mode 100644
index fcb1f48..0000000
--- a/apps/backend/app/services/link_analysis.py
+++ /dev/null
@@ -1,686 +0,0 @@
-"""
-Intelligent link prioritization and scoring inspired by crawl4ai.
-
-This module implements sophisticated link analysis capabilities:
-- 3-layer scoring system for smart link prioritization
-- Domain authority and credibility assessment
-- Content relevance scoring
-- Link quality metrics and filtering
-"""
-
-import asyncio
-import re
-import time
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Set, Any, Tuple
-from urllib.parse import urlparse, urljoin
-from collections import Counter, defaultdict
-
-import httpx
-import numpy as np
-from bs4 import BeautifulSoup
-import structlog
-
-from app.utils.text_processing import clean_tokens, sanitize_text
-
-logger = structlog.get_logger(__name__)
-
-
-@dataclass
-class LinkInfo:
-    """Comprehensive information about a link."""
-    url: str
-    text: str
-    title: Optional[str] = None
-    domain: str = ""
-    path: str = ""
-    is_external: bool = True
-    
-    # Scoring components
-    relevance_score: float = 0.0
-    authority_score: float = 0.0
-    quality_score: float = 0.0
-    freshness_score: float = 0.0
-    
-    # Combined scores
-    overall_score: float = 0.0
-    priority_rank: int = 0
-    
-    # Metadata
-    depth_from_root: int = 0
-    discovered_at: float = 0.0
-    extraction_context: Dict[str, Any] = field(default_factory=dict)
-    
-    def __post_init__(self):
-        if not self.domain and self.url:
-            parsed = urlparse(self.url)
-            self.domain = parsed.netloc
-            self.path = parsed.path
-        
-        if not self.discovered_at:
-            self.discovered_at = time.time()
-
-
-@dataclass
-class LinkPreviewConfig:
-    """Configuration for link analysis and scoring."""
-    query: Optional[str] = None
-    score_threshold: float = 0.3
-    concurrent_requests: int = 10
-    max_preview_length: int = 500
-    enable_domain_authority: bool = True
-    enable_content_preview: bool = True
-    enable_freshness_scoring: bool = True
-    preview_timeout: float = 5.0
-    
-    # Authority domain lists
-    high_authority_domains: List[str] = field(default_factory=lambda: [
-        'wikipedia.org', 'github.com', 'stackoverflow.com',
-        'mozilla.org', 'w3.org', 'ietf.org', 'arxiv.org',
-        'nature.com', 'sciencedirect.com', 'ieee.org'
-    ])
-    
-    medium_authority_domains: List[str] = field(default_factory=lambda: [
-        'medium.com', 'dev.to', 'reddit.com', 'news.ycombinator.com',
-        'techcrunch.com', 'arstechnica.com', 'wired.com'
-    ])
-    
-    low_quality_indicators: List[str] = field(default_factory=lambda: [
-        'ads', 'advertisement', 'popup', 'spam', 'click',
-        'buy-now', 'discount', 'offer', 'deal'
-    ])
-
-
-@dataclass
-class LinkAnalysisResult:
-    """Result of comprehensive link analysis."""
-    analyzed_links: List[LinkInfo]
-    top_links: List[LinkInfo]
-    analysis_metadata: Dict[str, Any]
-    domain_statistics: Dict[str, Any]
-    quality_distribution: Dict[str, int]
-
-
-class LinkScorer:
-    """
-    Sophisticated link scoring system with multiple scoring layers.
-    
-    Implements a 3-layer scoring approach:
-    1. Relevance scoring - content relevance to query
-    2. Authority scoring - domain credibility and authority
-    3. Quality scoring - link and content quality indicators
-    """
-    
-    def __init__(self, config: LinkPreviewConfig):
-        """Initialize link scorer with configuration."""
-        self.config = config
-        
-        # Compile regex patterns for performance
-        self.quality_patterns = {
-            'article_indicators': re.compile(r'(article|post|blog|news|story|guide|tutorial)', re.I),
-            'date_patterns': re.compile(r'(\d{4}[/-]\d{1,2}[/-]\d{1,2}|\d{1,2}[/-]\d{1,2}[/-]\d{4})'),
-            'low_quality': re.compile('|'.join(self.config.low_quality_indicators), re.I),
-            'file_extensions': re.compile(r'\.(pdf|doc|docx|ppt|pptx|xls|xlsx)$', re.I)
-        }
-    
-    async def score_links(
-        self,
-        links: List[str],
-        base_url: str = "",
-        link_texts: Optional[Dict[str, str]] = None,
-        **kwargs
-    ) -> List[LinkInfo]:
-        """
-        Score a list of links using the 3-layer scoring system.
-        
-        Args:
-            links: List of URLs to score
-            base_url: Base URL for context
-            link_texts: Optional mapping of URLs to their link text
-            **kwargs: Additional scoring parameters
-            
-        Returns:
-            List of LinkInfo objects with computed scores
-        """
-        start_time = time.time()
-        link_texts = link_texts or {}
-        
-        # Create LinkInfo objects
-        link_infos = []
-        for url in links:
-            absolute_url = urljoin(base_url, url) if base_url else url
-            text = link_texts.get(url, url)
-            
-            link_info = LinkInfo(
-                url=absolute_url,
-                text=text,
-                is_external=self._is_external_link(absolute_url, base_url)
-            )
-            link_infos.append(link_info)
-        
-        logger.info("link_scoring_started", total_links=len(link_infos))
-        
-        # Layer 1: Relevance Scoring
-        await self._score_relevance(link_infos)
-        
-        # Layer 2: Authority Scoring
-        await self._score_authority(link_infos)
-        
-        # Layer 3: Quality Scoring
-        await self._score_quality(link_infos)
-        
-        # Layer 4: Freshness Scoring (if enabled)
-        if self.config.enable_freshness_scoring:
-            await self._score_freshness(link_infos)
-        
-        # Compute overall scores and rankings
-        await self._compute_overall_scores(link_infos)
-        
-        # Sort by overall score
-        link_infos.sort(key=lambda x: x.overall_score, reverse=True)
-        
-        # Assign priority ranks
-        for i, link_info in enumerate(link_infos):
-            link_info.priority_rank = i + 1
-        
-        processing_time = time.time() - start_time
-        logger.info(
-            "link_scoring_completed",
-            total_links=len(link_infos),
-            processing_time=processing_time,
-            top_score=link_infos[0].overall_score if link_infos else 0
-        )
-        
-        return link_infos
-    
-    async def _score_relevance(self, link_infos: List[LinkInfo]):
-        """Layer 1: Score links based on relevance to query."""
-        if not self.config.query:
-            # No query provided, assign neutral relevance
-            for link_info in link_infos:
-                link_info.relevance_score = 0.5
-            return
-        
-        query_tokens = set(clean_tokens(self.config.query.lower().split()))
-        
-        for link_info in link_infos:
-            score = 0.0
-            
-            # Score based on URL path
-            url_tokens = set(clean_tokens(re.findall(r'[a-zA-Z]+', link_info.path.lower())))
-            if url_tokens:
-                url_overlap = len(query_tokens.intersection(url_tokens))
-                score += (url_overlap / len(query_tokens)) * 0.4
-            
-            # Score based on link text
-            if link_info.text and link_info.text != link_info.url:
-                text_tokens = set(clean_tokens(link_info.text.lower().split()))
-                if text_tokens:
-                    text_overlap = len(query_tokens.intersection(text_tokens))
-                    score += (text_overlap / len(query_tokens)) * 0.6
-            
-            # Bonus for exact query matches
-            if self.config.query.lower() in link_info.url.lower():
-                score += 0.3
-            if self.config.query.lower() in link_info.text.lower():
-                score += 0.4
-            
-            link_info.relevance_score = min(1.0, score)
-    
-    async def _score_authority(self, link_infos: List[LinkInfo]):
-        """Layer 2: Score links based on domain authority."""
-        for link_info in link_infos:
-            score = 0.5  # Base authority score
-            domain = link_info.domain.lower()
-            
-            # High authority domains
-            if any(auth_domain in domain for auth_domain in self.config.high_authority_domains):
-                score = 0.9
-            
-            # Medium authority domains
-            elif any(med_domain in domain for med_domain in self.config.medium_authority_domains):
-                score = 0.7
-            
-            # Educational and government domains
-            elif domain.endswith(('.edu', '.gov', '.org')):
-                score = 0.8
-            
-            # Well-known top-level domains
-            elif domain.endswith(('.com', '.net')):
-                score = 0.6
-            
-            # Country code TLDs
-            elif len(domain.split('.')[-1]) == 2:
-                score = 0.5
-            
-            # Subdomains penalty
-            if len(domain.split('.')) > 2:
-                score *= 0.9
-            
-            # Very short or very long domain names (potential spam indicators)
-            base_domain = domain.split('.')[0]
-            if len(base_domain) < 3 or len(base_domain) > 20:
-                score *= 0.8
-            
-            link_info.authority_score = score
-    
-    async def _score_quality(self, link_infos: List[LinkInfo]):
-        """Layer 3: Score links based on quality indicators."""
-        for link_info in link_infos:
-            score = 0.5  # Base quality score
-            url = link_info.url.lower()
-            path = link_info.path.lower()
-            text = link_info.text.lower()
-            
-            # Positive quality indicators
-            if self.quality_patterns['article_indicators'].search(path):
-                score += 0.2
-            
-            # File format bonuses
-            if self.quality_patterns['file_extensions'].search(url):
-                score += 0.15  # PDFs and documents often contain quality content
-            
-            # Link text quality
-            if link_info.text and link_info.text != link_info.url:
-                if len(link_info.text) > 10:  # Descriptive link text
-                    score += 0.15
-                
-                # Avoid generic link texts
-                generic_texts = ['click here', 'read more', 'more info', 'link', 'here']
-                if not any(generic in text for generic in generic_texts):
-                    score += 0.1
-            
-            # URL structure quality
-            if '?' not in url:  # Clean URLs without query parameters
-                score += 0.05
-            
-            if path.count('/') <= 4:  # Not too deep in site hierarchy
-                score += 0.05
-            
-            # Negative quality indicators
-            if self.quality_patterns['low_quality'].search(url) or self.quality_patterns['low_quality'].search(text):
-                score -= 0.3
-            
-            # Penalize very long URLs (potential spam)
-            if len(link_info.url) > 100:
-                score -= 0.1
-            
-            # Penalize URLs with many parameters
-            if url.count('&') > 5:
-                score -= 0.2
-            
-            link_info.quality_score = max(0.0, min(1.0, score))
-    
-    async def _score_freshness(self, link_infos: List[LinkInfo]):
-        """Layer 4: Score links based on freshness indicators."""
-        current_year = time.gmtime().tm_year
-        
-        for link_info in link_infos:
-            score = 0.5  # Base freshness score
-            
-            # Look for dates in URL
-            date_matches = self.quality_patterns['date_patterns'].findall(link_info.url)
-            if date_matches:
-                # Extract year from the most recent date found
-                years = []
-                for date_str in date_matches:
-                    # Simple year extraction
-                    year_match = re.search(r'(\d{4})', date_str)
-                    if year_match:
-                        year = int(year_match.group(1))
-                        if 2000 <= year <= current_year:  # Valid year range
-                            years.append(year)
-                
-                if years:
-                    latest_year = max(years)
-                    age = current_year - latest_year
-                    
-                    # Score based on age
-                    if age == 0:  # Current year
-                        score = 1.0
-                    elif age == 1:  # Last year
-                        score = 0.9
-                    elif age <= 3:  # Within 3 years
-                        score = 0.7
-                    elif age <= 5:  # Within 5 years
-                        score = 0.5
-                    else:  # Older content
-                        score = max(0.1, 0.5 - (age - 5) * 0.05)
-            
-            # Look for freshness indicators in path
-            fresh_indicators = ['2024', '2023', 'latest', 'new', 'recent', 'current']
-            path_lower = link_info.path.lower()
-            if any(indicator in path_lower for indicator in fresh_indicators):
-                score += 0.2
-            
-            link_info.freshness_score = min(1.0, score)
-    
-    async def _compute_overall_scores(self, link_infos: List[LinkInfo]):
-        """Compute overall scores by combining all scoring layers."""
-        # Weights for different score components
-        weights = {
-            'relevance': 0.35,
-            'authority': 0.25,
-            'quality': 0.25,
-            'freshness': 0.15
-        }
-        
-        for link_info in link_infos:
-            overall_score = (
-                link_info.relevance_score * weights['relevance'] +
-                link_info.authority_score * weights['authority'] +
-                link_info.quality_score * weights['quality'] +
-                link_info.freshness_score * weights['freshness']
-            )
-            
-            link_info.overall_score = overall_score
-            
-            # Store scoring breakdown for debugging
-            link_info.extraction_context = {
-                'relevance': link_info.relevance_score,
-                'authority': link_info.authority_score,
-                'quality': link_info.quality_score,
-                'freshness': link_info.freshness_score,
-                'weights': weights
-            }
-    
-    def _is_external_link(self, url: str, base_url: str) -> bool:
-        """Determine if a link is external to the base URL."""
-        if not base_url:
-            return True
-        
-        try:
-            url_domain = urlparse(url).netloc
-            base_domain = urlparse(base_url).netloc
-            return url_domain != base_domain
-        except:
-            return True
-
-
-class LinkAnalyzer:
-    """
-    Comprehensive link analysis system.
-    
-    This class orchestrates the complete link analysis process including
-    extraction, scoring, filtering, and preview generation.
-    """
-    
-    def __init__(self, config: LinkPreviewConfig):
-        """Initialize link analyzer."""
-        self.config = config
-        self.scorer = LinkScorer(config)
-    
-    async def analyze_page_links(
-        self,
-        html_content: str,
-        base_url: str = "",
-        **kwargs
-    ) -> LinkAnalysisResult:
-        """
-        Analyze all links found in HTML content.
-        
-        Args:
-            html_content: HTML content to analyze
-            base_url: Base URL for link resolution
-            **kwargs: Additional analysis parameters
-            
-        Returns:
-            LinkAnalysisResult with comprehensive link analysis
-        """
-        start_time = time.time()
-        
-        # Extract links from HTML
-        links, link_texts = self._extract_links_from_html(html_content, base_url)
-        
-        if not links:
-            return LinkAnalysisResult(
-                analyzed_links=[],
-                top_links=[],
-                analysis_metadata={'total_links': 0, 'error': 'No links found'},
-                domain_statistics={},
-                quality_distribution={}
-            )
-        
-        logger.info("link_analysis_started", total_links=len(links), base_url=base_url)
-        
-        # Score links
-        scored_links = await self.scorer.score_links(
-            links=links,
-            base_url=base_url,
-            link_texts=link_texts,
-            **kwargs
-        )
-        
-        # Filter links by score threshold
-        filtered_links = [
-            link for link in scored_links 
-            if link.overall_score >= self.config.score_threshold
-        ]
-        
-        # Generate previews for top links if enabled
-        if self.config.enable_content_preview:
-            top_links_for_preview = filtered_links[:self.config.concurrent_requests]
-            await self._generate_link_previews(top_links_for_preview)
-        
-        # Generate statistics
-        domain_stats = self._generate_domain_statistics(scored_links)
-        quality_dist = self._generate_quality_distribution(scored_links)
-        
-        processing_time = time.time() - start_time
-        
-        # Prepare metadata
-        analysis_metadata = {
-            'total_links_found': len(links),
-            'total_links_scored': len(scored_links),
-            'links_above_threshold': len(filtered_links),
-            'score_threshold': self.config.score_threshold,
-            'processing_time': processing_time,
-            'base_url': base_url,
-            'query': self.config.query,
-            'config': self.config.__dict__
-        }
-        
-        logger.info(
-            "link_analysis_completed",
-            total_analyzed=len(scored_links),
-            above_threshold=len(filtered_links),
-            processing_time=processing_time
-        )
-        
-        return LinkAnalysisResult(
-            analyzed_links=scored_links,
-            top_links=filtered_links,
-            analysis_metadata=analysis_metadata,
-            domain_statistics=domain_stats,
-            quality_distribution=quality_dist
-        )
-    
-    def _extract_links_from_html(
-        self, 
-        html_content: str, 
-        base_url: str
-    ) -> Tuple[List[str], Dict[str, str]]:
-        """Extract links and their text from HTML content."""
-        soup = BeautifulSoup(html_content, 'lxml')
-        links = []
-        link_texts = {}
-        
-        for a_tag in soup.find_all('a', href=True):
-            href = a_tag['href']
-            
-            # Skip anchor links, javascript, and email links
-            if href.startswith(('#', 'javascript:', 'mailto:')):
-                continue
-            
-            # Make URL absolute
-            absolute_url = urljoin(base_url, href)
-            
-            # Extract link text
-            link_text = sanitize_text(a_tag.get_text())
-            if not link_text:
-                link_text = a_tag.get('title', href)
-            
-            links.append(absolute_url)
-            link_texts[absolute_url] = link_text
-        
-        # Remove duplicates while preserving order
-        unique_links = []
-        seen = set()
-        for link in links:
-            if link not in seen:
-                unique_links.append(link)
-                seen.add(link)
-        
-        return unique_links, link_texts
-    
-    async def _generate_link_previews(self, links: List[LinkInfo]):
-        """Generate content previews for top links."""
-        if not links:
-            return
-        
-        logger.info("generating_link_previews", count=len(links))
-        
-        async def fetch_preview(link_info: LinkInfo):
-            try:
-                async with httpx.AsyncClient(timeout=self.config.preview_timeout) as client:
-                    response = await client.get(
-                        link_info.url,
-                        headers={
-                            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
-                        }
-                    )
-                    
-                    if response.status_code == 200:
-                        # Extract preview content
-                        soup = BeautifulSoup(response.text, 'lxml')
-                        
-                        # Try to get meta description
-                        meta_desc = soup.find('meta', attrs={'name': 'description'})
-                        if meta_desc and meta_desc.get('content'):
-                            preview = sanitize_text(meta_desc['content'])
-                        else:
-                            # Get first paragraph
-                            first_p = soup.find('p')
-                            preview = sanitize_text(first_p.get_text()) if first_p else ""
-                        
-                        # Truncate to max length
-                        if len(preview) > self.config.max_preview_length:
-                            preview = preview[:self.config.max_preview_length] + "..."
-                        
-                        link_info.extraction_context['preview'] = preview
-                        link_info.extraction_context['preview_status'] = 'success'
-                    
-            except Exception as e:
-                link_info.extraction_context['preview_error'] = str(e)
-                link_info.extraction_context['preview_status'] = 'failed'
-        
-        # Generate previews concurrently
-        tasks = [fetch_preview(link) for link in links]
-        await asyncio.gather(*tasks, return_exceptions=True)
-    
-    def _generate_domain_statistics(self, links: List[LinkInfo]) -> Dict[str, Any]:
-        """Generate statistics about domains in the link set."""
-        domain_counts = Counter(link.domain for link in links)
-        external_count = sum(1 for link in links if link.is_external)
-        internal_count = len(links) - external_count
-        
-        # Authority distribution
-        high_authority = sum(1 for link in links if link.authority_score >= 0.8)
-        medium_authority = sum(1 for link in links if 0.5 <= link.authority_score < 0.8)
-        low_authority = sum(1 for link in links if link.authority_score < 0.5)
-        
-        return {
-            'total_domains': len(domain_counts),
-            'most_common_domains': domain_counts.most_common(10),
-            'external_links': external_count,
-            'internal_links': internal_count,
-            'authority_distribution': {
-                'high': high_authority,
-                'medium': medium_authority,
-                'low': low_authority
-            }
-        }
-    
-    def _generate_quality_distribution(self, links: List[LinkInfo]) -> Dict[str, int]:
-        """Generate distribution of link quality scores."""
-        score_ranges = {
-            'excellent': 0,    # 0.8 - 1.0
-            'good': 0,         # 0.6 - 0.8
-            'average': 0,      # 0.4 - 0.6
-            'poor': 0,         # 0.2 - 0.4
-            'very_poor': 0     # 0.0 - 0.2
-        }
-        
-        for link in links:
-            score = link.overall_score
-            if score >= 0.8:
-                score_ranges['excellent'] += 1
-            elif score >= 0.6:
-                score_ranges['good'] += 1
-            elif score >= 0.4:
-                score_ranges['average'] += 1
-            elif score >= 0.2:
-                score_ranges['poor'] += 1
-            else:
-                score_ranges['very_poor'] += 1
-        
-        return score_ranges
-
-
-# Convenience functions
-async def analyze_links(
-    html_content: str,
-    base_url: str = "",
-    query: Optional[str] = None,
-    score_threshold: float = 0.3,
-    concurrent_requests: int = 10
-) -> LinkAnalysisResult:
-    """
-    Analyze links in HTML content with intelligent scoring.
-    
-    Args:
-        html_content: HTML content to analyze
-        base_url: Base URL for link resolution
-        query: Optional query for relevance scoring
-        score_threshold: Minimum score threshold for filtering
-        concurrent_requests: Number of concurrent preview requests
-        
-    Returns:
-        LinkAnalysisResult with comprehensive analysis
-    """
-    config = LinkPreviewConfig(
-        query=query,
-        score_threshold=score_threshold,
-        concurrent_requests=concurrent_requests
-    )
-    
-    analyzer = LinkAnalyzer(config)
-    return await analyzer.analyze_page_links(html_content, base_url)
-
-
-async def get_top_links(
-    html_content: str,
-    base_url: str = "",
-    query: Optional[str] = None,
-    top_k: int = 10
-) -> List[LinkInfo]:
-    """
-    Get top-scored links from HTML content.
-    
-    Args:
-        html_content: HTML content to analyze
-        base_url: Base URL for link resolution  
-        query: Optional query for relevance scoring
-        top_k: Number of top links to return
-        
-    Returns:
-        List of top-scored LinkInfo objects
-    """
-    result = await analyze_links(
-        html_content=html_content,
-        base_url=base_url,
-        query=query,
-        score_threshold=0.0  # No filtering, get all links
-    )
-    
-    return result.analyzed_links[:top_k]
diff --git a/apps/backend/app/services/link_preview.py b/apps/backend/app/services/link_preview.py
deleted file mode 100644
index 9682e16..0000000
--- a/apps/backend/app/services/link_preview.py
+++ /dev/null
@@ -1,672 +0,0 @@
-"""
-Advanced link preview system for extracting head content and metadata from links.
-
-This module provides sophisticated link processing capabilities:
-- Parallel link head extraction
-- Link filtering and scoring
-- Metadata extraction (title, description, images)
-- BM25 relevance scoring
-- Link quality assessment
-- Performance optimization with caching
-"""
-
-import asyncio
-import re
-from typing import Dict, List, Optional, Any, Set
-from urllib.parse import urljoin, urlparse
-from dataclasses import dataclass, field
-from datetime import datetime
-
-import httpx
-import structlog
-from bs4 import BeautifulSoup
-
-from app.utils.text_processing import clean_tokens, calculate_text_quality
-
-logger = structlog.get_logger(__name__)
-
-
-@dataclass
-class LinkMetadata:
-    """Metadata extracted from link head content."""
-    title: Optional[str] = None
-    description: Optional[str] = None
-    image: Optional[str] = None
-    url: Optional[str] = None
-    site_name: Optional[str] = None
-    type: Optional[str] = None
-    
-    # OpenGraph metadata
-    og_title: Optional[str] = None
-    og_description: Optional[str] = None
-    og_image: Optional[str] = None
-    og_url: Optional[str] = None
-    og_site_name: Optional[str] = None
-    og_type: Optional[str] = None
-    
-    # Twitter Card metadata
-    twitter_card: Optional[str] = None
-    twitter_title: Optional[str] = None
-    twitter_description: Optional[str] = None
-    twitter_image: Optional[str] = None
-    twitter_site: Optional[str] = None
-    twitter_creator: Optional[str] = None
-    
-    # Technical metadata
-    canonical_url: Optional[str] = None
-    language: Optional[str] = None
-    charset: Optional[str] = None
-    viewport: Optional[str] = None
-    
-    # Content analysis
-    content_preview: Optional[str] = None
-    content_length: int = 0
-    keywords: List[str] = field(default_factory=list)
-    
-    # Performance metrics
-    response_time: float = 0.0
-    status_code: Optional[int] = None
-    content_type: Optional[str] = None
-    
-    def get_best_title(self) -> Optional[str]:
-        """Get the best available title."""
-        return self.og_title or self.twitter_title or self.title
-    
-    def get_best_description(self) -> Optional[str]:
-        """Get the best available description."""
-        return self.og_description or self.twitter_description or self.description
-    
-    def get_best_image(self) -> Optional[str]:
-        """Get the best available image."""
-        return self.og_image or self.twitter_image or self.image
-
-
-@dataclass
-class LinkPreviewResult:
-    """Result of link preview extraction."""
-    url: str
-    success: bool
-    metadata: Optional[LinkMetadata] = None
-    error: Optional[str] = None
-    relevance_score: float = 0.0
-    quality_score: float = 0.0
-    processing_time: float = 0.0
-    
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary representation."""
-        result = {
-            'url': self.url,
-            'success': self.success,
-            'relevance_score': self.relevance_score,
-            'quality_score': self.quality_score,
-            'processing_time': self.processing_time
-        }
-        
-        if self.metadata:
-            result['metadata'] = {
-                'title': self.metadata.get_best_title(),
-                'description': self.metadata.get_best_description(),
-                'image': self.metadata.get_best_image(),
-                'site_name': self.metadata.og_site_name or self.metadata.site_name,
-                'canonical_url': self.metadata.canonical_url,
-                'language': self.metadata.language,
-                'content_preview': self.metadata.content_preview,
-                'keywords': self.metadata.keywords,
-                'status_code': self.metadata.status_code,
-                'content_type': self.metadata.content_type
-            }
-        
-        if self.error:
-            result['error'] = self.error
-        
-        return result
-
-
-@dataclass
-class LinkPreviewConfig:
-    """Configuration for link preview extraction."""
-    
-    # Filtering options
-    include_patterns: List[str] = field(default_factory=list)
-    exclude_patterns: List[str] = field(default_factory=list)
-    allowed_domains: List[str] = field(default_factory=list)
-    blocked_domains: List[str] = field(default_factory=list)
-    
-    # Processing options
-    concurrent_requests: int = 10
-    timeout: float = 10.0
-    max_content_length: int = 1024 * 1024  # 1MB
-    follow_redirects: bool = True
-    max_redirects: int = 5
-    
-    # Content options
-    extract_content_preview: bool = True
-    preview_length: int = 500
-    extract_keywords: bool = True
-    max_keywords: int = 20
-    
-    # Quality scoring
-    enable_quality_scoring: bool = True
-    enable_relevance_scoring: bool = True
-    query: Optional[str] = None
-    
-    # Performance options
-    enable_caching: bool = True
-    cache_ttl: int = 3600  # 1 hour
-
-
-class LinkPreview:
-    """
-    Advanced link preview system for extracting metadata and content from links.
-    
-    Provides intelligent link processing with filtering, scoring, and optimization.
-    """
-    
-    def __init__(self, config: LinkPreviewConfig = None):
-        """
-        Initialize link preview system.
-        
-        Args:
-            config: Configuration for link preview processing
-        """
-        self.config = config or LinkPreviewConfig()
-        self.client: Optional[httpx.AsyncClient] = None
-        
-        # Compile regex patterns for performance
-        self.include_patterns = [re.compile(p) for p in self.config.include_patterns]
-        self.exclude_patterns = [re.compile(p) for p in self.config.exclude_patterns]
-        
-        # Query tokens for relevance scoring
-        self.query_tokens = set(clean_tokens(self.config.query.lower().split())) if self.config.query else set()
-        
-        # Cache for results
-        self._cache: Dict[str, LinkPreviewResult] = {}
-    
-    async def __aenter__(self):
-        """Async context manager entry."""
-        await self.initialize()
-        return self
-    
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        await self.close()
-    
-    async def initialize(self):
-        """Initialize HTTP client."""
-        if not self.client:
-            self.client = httpx.AsyncClient(
-                timeout=httpx.Timeout(self.config.timeout),
-                limits=httpx.Limits(
-                    max_connections=self.config.concurrent_requests * 2,
-                    max_keepalive_connections=self.config.concurrent_requests
-                ),
-                follow_redirects=self.config.follow_redirects,
-                max_redirects=self.config.max_redirects,
-                headers={
-                    'User-Agent': 'Mozilla/5.0 (compatible; LinkPreview/1.0; +https://example.com/bot)',
-                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                    'Accept-Language': 'en-US,en;q=0.5',
-                    'Accept-Encoding': 'gzip, deflate',
-                    'DNT': '1',
-                    'Connection': 'keep-alive'
-                }
-            )
-    
-    async def close(self):
-        """Close HTTP client."""
-        if self.client:
-            await self.client.aclose()
-            self.client = None
-    
-    def _should_process_link(self, url: str) -> bool:
-        """Check if link should be processed based on filters."""
-        # Check include patterns
-        if self.include_patterns:
-            if not any(pattern.search(url) for pattern in self.include_patterns):
-                return False
-        
-        # Check exclude patterns
-        if self.exclude_patterns:
-            if any(pattern.search(url) for pattern in self.exclude_patterns):
-                return False
-        
-        # Check domain filters
-        parsed = urlparse(url)
-        domain = parsed.netloc.lower()
-        
-        # Check blocked domains
-        if self.config.blocked_domains:
-            if any(blocked in domain for blocked in self.config.blocked_domains):
-                return False
-        
-        # Check allowed domains
-        if self.config.allowed_domains:
-            if not any(allowed in domain for allowed in self.config.allowed_domains):
-                return False
-        
-        return True
-    
-    async def extract_link_previews(self, links: List[str]) -> List[LinkPreviewResult]:
-        """
-        Extract previews for multiple links.
-        
-        Args:
-            links: List of URLs to process
-            
-        Returns:
-            List of LinkPreviewResult objects
-        """
-        if not self.client:
-            await self.initialize()
-        
-        # Filter links
-        filtered_links = [link for link in links if self._should_process_link(link)]
-        
-        logger.info(f"Processing {len(filtered_links)} links (filtered from {len(links)})")
-        
-        if not filtered_links:
-            return []
-        
-        # Process links with concurrency control
-        semaphore = asyncio.Semaphore(self.config.concurrent_requests)
-        
-        async def process_link(url: str) -> LinkPreviewResult:
-            async with semaphore:
-                return await self._extract_single_preview(url)
-        
-        # Execute all requests
-        results = await asyncio.gather(
-            *[process_link(url) for url in filtered_links],
-            return_exceptions=True
-        )
-        
-        # Handle exceptions
-        processed_results = []
-        for i, result in enumerate(results):
-            if isinstance(result, Exception):
-                error_result = LinkPreviewResult(
-                    url=filtered_links[i],
-                    success=False,
-                    error=str(result)
-                )
-                processed_results.append(error_result)
-            else:
-                processed_results.append(result)
-        
-        # Sort by quality and relevance scores
-        processed_results.sort(
-            key=lambda x: (x.relevance_score, x.quality_score), 
-            reverse=True
-        )
-        
-        logger.info(f"Completed link preview extraction: {len(processed_results)} results")
-        
-        return processed_results
-    
-    async def _extract_single_preview(self, url: str) -> LinkPreviewResult:
-        """Extract preview for a single link."""
-        import time
-        start_time = time.time()
-        
-        # Check cache
-        if self.config.enable_caching and url in self._cache:
-            cached_result = self._cache[url]
-            cached_result.processing_time = time.time() - start_time
-            return cached_result
-        
-        try:
-            # Make HEAD request first to check content type and size
-            head_response = await self.client.head(url)
-            content_type = head_response.headers.get('content-type', '').lower()
-            content_length = int(head_response.headers.get('content-length', 0))
-            
-            # Skip if not HTML-like content
-            if not ('text/html' in content_type or 'application/xhtml' in content_type):
-                result = LinkPreviewResult(
-                    url=url,
-                    success=False,
-                    error=f"Unsupported content type: {content_type}",
-                    processing_time=time.time() - start_time
-                )
-                return result
-            
-            # Skip if content too large
-            if content_length > self.config.max_content_length:
-                result = LinkPreviewResult(
-                    url=url,
-                    success=False,
-                    error=f"Content too large: {content_length} bytes",
-                    processing_time=time.time() - start_time
-                )
-                return result
-            
-            # Make GET request for content
-            response = await self.client.get(url)
-            response.raise_for_status()
-            
-            # Extract metadata from HTML
-            metadata = self._extract_metadata_from_html(response.text, url)
-            metadata.status_code = response.status_code
-            metadata.content_type = content_type
-            metadata.response_time = time.time() - start_time
-            
-            # Calculate scores
-            relevance_score = self._calculate_relevance_score(metadata) if self.config.enable_relevance_scoring else 0.0
-            quality_score = self._calculate_quality_score(metadata) if self.config.enable_quality_scoring else 0.0
-            
-            result = LinkPreviewResult(
-                url=url,
-                success=True,
-                metadata=metadata,
-                relevance_score=relevance_score,
-                quality_score=quality_score,
-                processing_time=time.time() - start_time
-            )
-            
-            # Cache result
-            if self.config.enable_caching:
-                self._cache[url] = result
-            
-            return result
-            
-        except httpx.HTTPError as e:
-            result = LinkPreviewResult(
-                url=url,
-                success=False,
-                error=f"HTTP error: {str(e)}",
-                processing_time=time.time() - start_time
-            )
-            return result
-        
-        except Exception as e:
-            result = LinkPreviewResult(
-                url=url,
-                success=False,
-                error=f"Processing error: {str(e)}",
-                processing_time=time.time() - start_time
-            )
-            return result
-    
-    def _extract_metadata_from_html(self, html: str, base_url: str) -> LinkMetadata:
-        """Extract metadata from HTML content."""
-        soup = BeautifulSoup(html, 'lxml')
-        metadata = LinkMetadata()
-        
-        # Basic metadata
-        title_tag = soup.find('title')
-        if title_tag:
-            metadata.title = title_tag.get_text().strip()
-        
-        # Meta tags
-        for meta in soup.find_all('meta'):
-            name = meta.get('name', '').lower()
-            property_attr = meta.get('property', '').lower()
-            content = meta.get('content', '').strip()
-            
-            if not content:
-                continue
-            
-            # Standard meta tags
-            if name == 'description':
-                metadata.description = content
-            elif name == 'keywords':
-                metadata.keywords = [kw.strip() for kw in content.split(',')]
-            elif name == 'author':
-                pass  # Could add author field
-            
-            # OpenGraph tags
-            elif property_attr.startswith('og:'):
-                og_type = property_attr[3:]  # Remove 'og:' prefix
-                if og_type == 'title':
-                    metadata.og_title = content
-                elif og_type == 'description':
-                    metadata.og_description = content
-                elif og_type == 'image':
-                    metadata.og_image = urljoin(base_url, content)
-                elif og_type == 'url':
-                    metadata.og_url = content
-                elif og_type == 'site_name':
-                    metadata.og_site_name = content
-                elif og_type == 'type':
-                    metadata.og_type = content
-            
-            # Twitter Card tags
-            elif name.startswith('twitter:'):
-                twitter_type = name[8:]  # Remove 'twitter:' prefix
-                if twitter_type == 'card':
-                    metadata.twitter_card = content
-                elif twitter_type == 'title':
-                    metadata.twitter_title = content
-                elif twitter_type == 'description':
-                    metadata.twitter_description = content
-                elif twitter_type == 'image':
-                    metadata.twitter_image = urljoin(base_url, content)
-                elif twitter_type == 'site':
-                    metadata.twitter_site = content
-                elif twitter_type == 'creator':
-                    metadata.twitter_creator = content
-            
-            # Technical metadata
-            elif name == 'viewport':
-                metadata.viewport = content
-            elif property_attr == 'charset' or name == 'charset':
-                metadata.charset = content
-        
-        # Canonical URL
-        canonical = soup.find('link', rel='canonical')
-        if canonical and canonical.get('href'):
-            metadata.canonical_url = urljoin(base_url, canonical['href'])
-        
-        # Language
-        html_tag = soup.find('html')
-        if html_tag:
-            metadata.language = html_tag.get('lang')
-        
-        # Extract content preview
-        if self.config.extract_content_preview:
-            content_preview = self._extract_content_preview(soup)
-            metadata.content_preview = content_preview[:self.config.preview_length] if content_preview else None
-        
-        # Extract keywords if not found in meta tags
-        if self.config.extract_keywords and not metadata.keywords:
-            metadata.keywords = self._extract_keywords_from_content(soup)
-        
-        # Content length
-        body = soup.find('body')
-        if body:
-            metadata.content_length = len(body.get_text())
-        
-        return metadata
-    
-    def _extract_content_preview(self, soup: BeautifulSoup) -> Optional[str]:
-        """Extract preview text from page content."""
-        # Try to find main content areas
-        main_selectors = [
-            'main', 'article', '.content', '.main-content', 
-            '.post-content', '.entry-content', '#content'
-        ]
-        
-        content_text = ""
-        
-        for selector in main_selectors:
-            elements = soup.select(selector)
-            if elements:
-                for element in elements:
-                    text = element.get_text().strip()
-                    if len(text) > len(content_text):
-                        content_text = text
-                break
-        
-        # Fallback to body text
-        if not content_text:
-            body = soup.find('body')
-            if body:
-                content_text = body.get_text()
-        
-        if content_text:
-            # Clean up text
-            content_text = re.sub(r'\s+', ' ', content_text).strip()
-            return content_text
-        
-        return None
-    
-    def _extract_keywords_from_content(self, soup: BeautifulSoup) -> List[str]:
-        """Extract keywords from page content."""
-        # Get text from important elements
-        text_elements = []
-        
-        # Headers
-        for header in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
-            text_elements.append(header.get_text())
-        
-        # Bold and italic text
-        for emphasis in soup.find_all(['b', 'strong', 'i', 'em']):
-            text_elements.append(emphasis.get_text())
-        
-        # First paragraph
-        first_p = soup.find('p')
-        if first_p:
-            text_elements.append(first_p.get_text())
-        
-        # Extract meaningful words
-        all_text = ' '.join(text_elements).lower()
-        words = re.findall(r'\b[a-zA-Z]{3,}\b', all_text)
-        
-        # Filter common stop words
-        stop_words = {
-            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 
-            'with', 'by', 'this', 'that', 'these', 'those', 'is', 'are', 'was', 'were',
-            'will', 'would', 'could', 'should', 'may', 'might', 'can', 'have', 'has',
-            'had', 'do', 'does', 'did', 'get', 'got', 'make', 'made', 'take', 'took'
-        }
-        
-        filtered_words = [word for word in words if word not in stop_words]
-        
-        # Count frequency and return top keywords
-        from collections import Counter
-        word_counts = Counter(filtered_words)
-        
-        return [word for word, count in word_counts.most_common(self.config.max_keywords)]
-    
-    def _calculate_relevance_score(self, metadata: LinkMetadata) -> float:
-        """Calculate relevance score based on query."""
-        if not self.query_tokens:
-            return 0.5
-        
-        # Combine all textual content
-        text_content = ' '.join(filter(None, [
-            metadata.get_best_title(),
-            metadata.get_best_description(),
-            metadata.content_preview,
-            ' '.join(metadata.keywords)
-        ])).lower()
-        
-        if not text_content:
-            return 0.0
-        
-        # Extract words from content
-        content_words = set(re.findall(r'\b[a-zA-Z]+\b', text_content))
-        
-        # Calculate overlap with query
-        overlap = len(self.query_tokens.intersection(content_words))
-        max_possible = len(self.query_tokens)
-        
-        return overlap / max_possible if max_possible > 0 else 0.0
-    
-    def _calculate_quality_score(self, metadata: LinkMetadata) -> float:
-        """Calculate quality score based on metadata completeness and content."""
-        score = 0.0
-        
-        # Metadata completeness (0.4 weight)
-        metadata_score = 0.0
-        if metadata.get_best_title():
-            metadata_score += 0.25
-        if metadata.get_best_description():
-            metadata_score += 0.25
-        if metadata.get_best_image():
-            metadata_score += 0.15
-        if metadata.canonical_url:
-            metadata_score += 0.1
-        if metadata.language:
-            metadata_score += 0.05
-        if metadata.keywords:
-            metadata_score += 0.2
-        
-        score += metadata_score * 0.4
-        
-        # Content quality (0.3 weight)
-        content_score = 0.0
-        if metadata.content_preview:
-            preview_length = len(metadata.content_preview)
-            # Optimal preview length around 200-300 chars
-            if 100 <= preview_length <= 500:
-                content_score += 0.3
-            elif preview_length > 50:
-                content_score += 0.2
-            else:
-                content_score += 0.1
-        
-        if metadata.content_length > 0:
-            # Content length scoring (sweet spot around 1000-5000 chars)
-            if 1000 <= metadata.content_length <= 10000:
-                content_score += 0.2
-            elif metadata.content_length > 500:
-                content_score += 0.1
-        
-        score += content_score * 0.3
-        
-        # Technical quality (0.3 weight)
-        technical_score = 0.0
-        if metadata.status_code == 200:
-            technical_score += 0.4
-        elif metadata.status_code and 200 <= metadata.status_code < 300:
-            technical_score += 0.3
-        
-        if metadata.response_time and metadata.response_time < 2.0:
-            technical_score += 0.3
-        elif metadata.response_time and metadata.response_time < 5.0:
-            technical_score += 0.1
-        
-        if metadata.content_type and 'html' in metadata.content_type:
-            technical_score += 0.3
-        
-        score += technical_score * 0.3
-        
-        return min(1.0, score)
-    
-    def get_cache_stats(self) -> Dict[str, Any]:
-        """Get cache statistics."""
-        return {
-            'cache_size': len(self._cache),
-            'cache_enabled': self.config.enable_caching,
-            'cache_ttl': self.config.cache_ttl
-        }
-    
-    def clear_cache(self):
-        """Clear the preview cache."""
-        self._cache.clear()
-        logger.info("Link preview cache cleared")
-
-
-# Convenience functions
-async def extract_link_previews(
-    links: List[str],
-    config: LinkPreviewConfig = None
-) -> List[LinkPreviewResult]:
-    """Extract link previews with default configuration."""
-    async with LinkPreview(config) as previewer:
-        return await previewer.extract_link_previews(links)
-
-
-def filter_links_by_quality(
-    results: List[LinkPreviewResult],
-    min_quality_score: float = 0.3,
-    min_relevance_score: float = 0.2
-) -> List[LinkPreviewResult]:
-    """Filter links by quality and relevance scores."""
-    return [
-        result for result in results
-        if result.success and 
-           result.quality_score >= min_quality_score and 
-           result.relevance_score >= min_relevance_score
-    ]
diff --git a/apps/backend/app/services/llm_configuration.py b/apps/backend/app/services/llm_configuration.py
deleted file mode 100644
index f5b41c3..0000000
--- a/apps/backend/app/services/llm_configuration.py
+++ /dev/null
@@ -1,503 +0,0 @@
-"""
-LLM-powered configuration service for converting natural language to crawler options.
-
-Inspired by Firecrawl's natural language configuration capabilities.
-"""
-
-import asyncio
-import json
-from typing import Dict, List, Optional, Any, Union
-from dataclasses import dataclass
-from enum import Enum
-import structlog
-import openai
-
-from app.config import get_settings
-from app.models.requests import ScrapingConfig, ExtractionStrategyConfig, ContentFilterConfig
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class ConfigurationPromptType(Enum):
-    """Types of configuration prompts."""
-    CRAWLER_OPTIONS = "crawler_options"
-    EXTRACTION_SCHEMA = "extraction_schema" 
-    CONTENT_FILTER = "content_filter"
-    SEARCH_STRATEGY = "search_strategy"
-
-
-@dataclass
-class LLMConfigurationRequest:
-    """Request for LLM-powered configuration generation."""
-    prompt: str
-    prompt_type: ConfigurationPromptType
-    context: Optional[Dict[str, Any]] = None
-    model: str = "gpt-4"
-    temperature: float = 0.1
-    max_retries: int = 3
-
-
-@dataclass
-class LLMConfigurationResponse:
-    """Response from LLM configuration generation."""
-    success: bool
-    config: Optional[Dict[str, Any]] = None
-    reasoning: Optional[str] = None
-    error: Optional[str] = None
-    tokens_used: int = 0
-    model_used: str = ""
-
-
-class LLMConfigurationService:
-    """
-    Service for converting natural language descriptions into structured configurations.
-    
-    Provides intelligent configuration generation for:
-    - Crawler options and settings
-    - Data extraction schemas
-    - Content filtering rules
-    - Search strategies
-    """
-    
-    def __init__(self):
-        """Initialize LLM configuration service."""
-        self.openai_client = openai.AsyncOpenAI(
-            api_key=getattr(settings, 'openai_api_key', None)
-        )
-        self.model_configs = {
-            "gpt-4": {"max_tokens": 4096, "fallback": "gpt-3.5-turbo"},
-            "gpt-4-turbo": {"max_tokens": 4096, "fallback": "gpt-4"},
-            "gpt-3.5-turbo": {"max_tokens": 4096, "fallback": None}
-        }
-        self.usage_stats = {"total_requests": 0, "successful_requests": 0, "tokens_used": 0}
-    
-    async def generate_config(self, request: LLMConfigurationRequest) -> LLMConfigurationResponse:
-        """
-        Generate configuration from natural language prompt.
-        
-        Args:
-            request: Configuration request with prompt and type
-            
-        Returns:
-            LLMConfigurationResponse with generated configuration
-        """
-        self.usage_stats["total_requests"] += 1
-        
-        logger.info("llm_config_generation_started",
-                   prompt=request.prompt[:100] + "..." if len(request.prompt) > 100 else request.prompt,
-                   type=request.prompt_type.value)
-        
-        # Get appropriate system prompt
-        system_prompt = self._get_system_prompt(request.prompt_type)
-        
-        # Try with primary model, fallback if needed
-        models_to_try = [request.model]
-        if request.model in self.model_configs and self.model_configs[request.model]["fallback"]:
-            models_to_try.append(self.model_configs[request.model]["fallback"])
-        
-        last_error = None
-        
-        for model in models_to_try:
-            for attempt in range(request.max_retries):
-                try:
-                    response = await self._make_llm_request(
-                        system_prompt=system_prompt,
-                        user_prompt=request.prompt,
-                        model=model,
-                        temperature=request.temperature + (attempt * 0.1),  # Increase temp on retries
-                        context=request.context
-                    )
-                    
-                    if response.success:
-                        self.usage_stats["successful_requests"] += 1
-                        self.usage_stats["tokens_used"] += response.tokens_used
-                        return response
-                    else:
-                        last_error = response.error
-                        
-                except Exception as e:
-                    last_error = str(e)
-                    logger.warning("llm_request_failed", 
-                                 model=model, 
-                                 attempt=attempt + 1, 
-                                 error=str(e))
-                    
-                    # Add delay before retry
-                    await asyncio.sleep(1 + attempt)
-        
-        return LLMConfigurationResponse(
-            success=False,
-            error=f"Failed to generate configuration: {last_error}"
-        )
-    
-    async def _make_llm_request(
-        self,
-        system_prompt: str,
-        user_prompt: str,
-        model: str,
-        temperature: float,
-        context: Optional[Dict[str, Any]] = None
-    ) -> LLMConfigurationResponse:
-        """Make request to LLM API."""
-        
-        # Prepare messages
-        messages = [
-            {"role": "system", "content": system_prompt}
-        ]
-        
-        # Add context if provided
-        if context:
-            context_prompt = f"Additional context: {json.dumps(context, indent=2)}\n\n"
-            user_prompt = context_prompt + user_prompt
-        
-        messages.append({"role": "user", "content": user_prompt})
-        
-        # Make API request
-        response = await self.openai_client.chat.completions.create(
-            model=model,
-            messages=messages,
-            temperature=temperature,
-            max_tokens=2000,
-            response_format={"type": "json_object"}
-        )
-        
-        # Process response
-        try:
-            content = response.choices[0].message.content
-            config_data = json.loads(content)
-            
-            return LLMConfigurationResponse(
-                success=True,
-                config=config_data.get("config", {}),
-                reasoning=config_data.get("reasoning", ""),
-                tokens_used=response.usage.total_tokens,
-                model_used=model
-            )
-            
-        except (json.JSONDecodeError, KeyError) as e:
-            return LLMConfigurationResponse(
-                success=False,
-                error=f"Failed to parse LLM response: {str(e)}"
-            )
-    
-    def _get_system_prompt(self, prompt_type: ConfigurationPromptType) -> str:
-        """Get appropriate system prompt for configuration type."""
-        
-        if prompt_type == ConfigurationPromptType.CRAWLER_OPTIONS:
-            return self._get_crawler_options_prompt()
-        elif prompt_type == ConfigurationPromptType.EXTRACTION_SCHEMA:
-            return self._get_extraction_schema_prompt()
-        elif prompt_type == ConfigurationPromptType.CONTENT_FILTER:
-            return self._get_content_filter_prompt()
-        elif prompt_type == ConfigurationPromptType.SEARCH_STRATEGY:
-            return self._get_search_strategy_prompt()
-        else:
-            return "You are a helpful assistant that generates structured configurations from natural language."
-    
-    def _get_crawler_options_prompt(self) -> str:
-        """System prompt for crawler options generation."""
-        return """You are a web crawler configuration expert. Generate crawler options based on natural language instructions.
-
-Available crawler options:
-- includePaths: string[] - URL pathname regex patterns that include matching URLs in the crawl. Only the paths that match the specified patterns will be included in the response. For example, if you set "includePaths": ["blog/.*"] for the base URL firecrawl.dev, only results matching that pattern will be included, such as https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap.
-- excludePaths: string[] - URL pathname regex patterns that exclude matching URLs from the crawl. For example, if you set "excludePaths": ["blog/.*"] for the base URL firecrawl.dev, any results matching that pattern will be excluded, such as https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap.
-- maxDepth: number - Maximum absolute depth to crawl from the base of the entered URL. Basically, the max number of slashes the pathname of a scraped URL may contain. Default: 10
-- maxDiscoveryDepth: number - Maximum depth to crawl based on discovery order. The root site and sitemapped pages has a discovery depth of 0. For example, if you set it to 1, and you set ignoreSitemap, you will only crawl the entered URL and all URLs that are linked on that page.
-- crawlEntireDomain: boolean - Allows the crawler to follow internal links to sibling or parent URLs, not just child paths. false: Only crawls deeper (child) URLs. → e.g. /features/feature-1 → /features/feature-1/tips ✅ → Won't follow /pricing or / ❌. true: Crawls any internal links, including siblings and parents. → e.g. /features/feature-1 → /pricing, /, etc. ✅. Use true for broader internal coverage beyond nested paths. Default: false
-- allowExternalLinks: boolean - Allows the crawler to follow links to external websites. Default: false
-- allowSubdomains: boolean - Allows the crawler to follow links to subdomains of the main domain. Default: false
-- sitemap: "skip" | "include" - Whether to ignore sitemap. Default: "include"
-- ignoreQueryParameters: boolean - Do not re-scrape the same path with different (or none) query parameters. Default: false
-- deduplicateSimilarURLs: boolean - Whether to deduplicate similar URLs
-- delay: number - Delay in seconds between scrapes. This helps respect website rate limits.
-- limit: number - Maximum number of pages to crawl. Default limit is 10000.
-- javascript_rendering: boolean - Whether to enable JavaScript rendering for dynamic content
-- screenshot: boolean - Whether to take screenshots of pages
-- extract_images: boolean - Whether to extract images from pages
-- extract_links: boolean - Whether to extract links from pages
-- stealth_mode: boolean - Whether to use stealth mode to avoid detection
-- mobile_mode: boolean - Whether to simulate mobile browser
-- wait_time: number - Time to wait after page load (in seconds)
-
-Return a JSON object with only the relevant options for the user's request. Don't include options that aren't relevant to the instruction. Focus on the most important options that directly address the user's intent.
-
-Response format:
-{
-  "config": {
-    "includePaths": ["pattern1", "pattern2"],
-    "maxDepth": 5,
-    "javascript_rendering": true,
-    // ... other relevant options
-  },
-  "reasoning": "Explanation of why these options were chosen for the user's request."
-}"""
-    
-    def _get_extraction_schema_prompt(self) -> str:
-        """System prompt for extraction schema generation."""
-        return """You are a data extraction expert. Generate JSON schemas for extracting structured data from web pages based on natural language descriptions.
-
-You can create schemas for extracting:
-- Articles and blog posts (title, content, author, date, tags)
-- Product information (name, price, description, specifications, reviews)
-- Contact information (name, email, phone, address)
-- Job listings (title, company, location, salary, requirements)
-- Event information (name, date, location, description)
-- Research papers (title, authors, abstract, publication date)
-- News articles (headline, summary, author, publication date)
-- Business listings (name, address, phone, website, reviews)
-- Social media posts (content, author, timestamp, engagement)
-- And any other structured data
-
-The schema should follow JSON Schema format with appropriate types, descriptions, and validation rules.
-
-Response format:
-{
-  "config": {
-    "type": "object",
-    "properties": {
-      "title": {
-        "type": "string",
-        "description": "The title or headline"
-      },
-      "content": {
-        "type": "string", 
-        "description": "The main content or body text"
-      }
-      // ... other fields
-    },
-    "required": ["title", "content"]
-  },
-  "reasoning": "Explanation of the schema design and why these fields are important."
-}"""
-    
-    def _get_content_filter_prompt(self) -> str:
-        """System prompt for content filter generation."""
-        return """You are a content filtering expert. Generate content filter configurations based on natural language requirements.
-
-Available filter types:
-- pruning: Remove irrelevant content based on configurable thresholds
-  - Options: relevance_threshold (0.0-1.0), content_length_min, content_length_max
-- bm25: Information retrieval-based filtering using BM25 algorithm
-  - Options: query_terms, score_threshold, max_results
-- llm: AI-powered content relevance filtering
-  - Options: relevance_query, confidence_threshold, model
-
-Filter configurations:
-{
-  "filter_type": "pruning|bm25|llm",
-  "config": {
-    // Type-specific options
-  }
-}
-
-Response format:
-{
-  "config": {
-    "filter_type": "bm25",
-    "config": {
-      "query_terms": ["technology", "AI"],
-      "score_threshold": 0.5,
-      "max_results": 50
-    }
-  },
-  "reasoning": "Explanation of why this filter configuration matches the user's requirements."
-}"""
-    
-    def _get_search_strategy_prompt(self) -> str:
-        """System prompt for search strategy generation."""
-        return """You are a search strategy expert. Generate search configurations based on natural language requirements.
-
-Available search options:
-- engines: ["google", "bing", "searxng", "duckduckgo"] - Search engines to use
-- max_results: number - Maximum number of results to return
-- language: string - Language code (en, es, fr, etc.)
-- country: string - Country code (us, uk, de, etc.)
-- safe_search: "strict" | "moderate" | "off" - Safe search setting
-- time_filter: "day" | "week" | "month" | "year" - Time-based filtering
-- result_type: "web" | "images" | "news" | "videos" - Type of results
-- advanced_operators: string[] - Advanced search operators to include
-
-Response format:
-{
-  "config": {
-    "engines": ["google", "searxng"],
-    "max_results": 20,
-    "language": "en",
-    "country": "us",
-    "safe_search": "moderate",
-    "advanced_operators": ["site:example.com", "intitle:AI"]
-  },
-  "reasoning": "Explanation of the search strategy and why these settings were chosen."
-}"""
-    
-    async def generate_crawler_options(self, prompt: str, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
-        """Generate crawler options from natural language prompt."""
-        request = LLMConfigurationRequest(
-            prompt=prompt,
-            prompt_type=ConfigurationPromptType.CRAWLER_OPTIONS,
-            context=context
-        )
-        
-        response = await self.generate_config(request)
-        return response.config if response.success else {}
-    
-    async def generate_extraction_schema(self, prompt: str, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
-        """Generate extraction schema from natural language prompt."""
-        request = LLMConfigurationRequest(
-            prompt=prompt,
-            prompt_type=ConfigurationPromptType.EXTRACTION_SCHEMA,
-            context=context
-        )
-        
-        response = await self.generate_config(request)
-        return response.config if response.success else {}
-    
-    async def generate_content_filter(self, prompt: str, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
-        """Generate content filter configuration from natural language prompt."""
-        request = LLMConfigurationRequest(
-            prompt=prompt,
-            prompt_type=ConfigurationPromptType.CONTENT_FILTER,
-            context=context
-        )
-        
-        response = await self.generate_config(request)
-        return response.config if response.success else {}
-    
-    async def generate_search_strategy(self, prompt: str, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
-        """Generate search strategy from natural language prompt."""
-        request = LLMConfigurationRequest(
-            prompt=prompt,
-            prompt_type=ConfigurationPromptType.SEARCH_STRATEGY,
-            context=context
-        )
-        
-        response = await self.generate_config(request)
-        return response.config if response.success else {}
-    
-    async def get_usage_stats(self) -> Dict[str, Any]:
-        """Get service usage statistics."""
-        return {
-            "usage_stats": self.usage_stats,
-            "success_rate": (
-                self.usage_stats["successful_requests"] / max(1, self.usage_stats["total_requests"])
-            ),
-            "average_tokens_per_request": (
-                self.usage_stats["tokens_used"] / max(1, self.usage_stats["successful_requests"])
-            )
-        }
-
-
-class ConfigurationValidator:
-    """Validates generated configurations for correctness and safety."""
-    
-    @staticmethod
-    def validate_crawler_options(config: Dict[str, Any]) -> Tuple[bool, List[str]]:
-        """Validate crawler options configuration."""
-        errors = []
-        
-        # Validate numeric ranges
-        if "maxDepth" in config and not (1 <= config["maxDepth"] <= 20):
-            errors.append("maxDepth must be between 1 and 20")
-        
-        if "limit" in config and not (1 <= config["limit"] <= 50000):
-            errors.append("limit must be between 1 and 50000")
-        
-        if "delay" in config and not (0 <= config["delay"] <= 60):
-            errors.append("delay must be between 0 and 60 seconds")
-        
-        # Validate boolean options
-        boolean_fields = ["crawlEntireDomain", "allowExternalLinks", "allowSubdomains", 
-                         "javascript_rendering", "screenshot", "extract_images", 
-                         "extract_links", "stealth_mode", "mobile_mode"]
-        
-        for field in boolean_fields:
-            if field in config and not isinstance(config[field], bool):
-                errors.append(f"{field} must be a boolean value")
-        
-        # Validate regex patterns
-        if "includePaths" in config:
-            if not isinstance(config["includePaths"], list):
-                errors.append("includePaths must be an array")
-            else:
-                for i, pattern in enumerate(config["includePaths"]):
-                    try:
-                        import re
-                        re.compile(pattern)
-                    except re.error:
-                        errors.append(f"includePaths[{i}] is not a valid regex pattern")
-        
-        return len(errors) == 0, errors
-    
-    @staticmethod
-    def validate_extraction_schema(config: Dict[str, Any]) -> Tuple[bool, List[str]]:
-        """Validate extraction schema configuration."""
-        errors = []
-        
-        # Basic JSON Schema validation
-        if "type" not in config:
-            errors.append("Schema must have a 'type' field")
-        elif config["type"] != "object":
-            errors.append("Schema type must be 'object'")
-        
-        if "properties" not in config:
-            errors.append("Schema must have a 'properties' field")
-        elif not isinstance(config["properties"], dict):
-            errors.append("Schema 'properties' must be an object")
-        
-        # Validate property definitions
-        if "properties" in config:
-            for prop_name, prop_def in config["properties"].items():
-                if not isinstance(prop_def, dict):
-                    errors.append(f"Property '{prop_name}' must be an object")
-                elif "type" not in prop_def:
-                    errors.append(f"Property '{prop_name}' must have a 'type' field")
-        
-        return len(errors) == 0, errors
-
-
-# Singleton instance
-_llm_config_service: Optional[LLMConfigurationService] = None
-
-
-async def get_llm_config_service() -> LLMConfigurationService:
-    """Get or create LLM configuration service instance."""
-    global _llm_config_service
-    
-    if _llm_config_service is None:
-        _llm_config_service = LLMConfigurationService()
-    
-    return _llm_config_service
-
-
-# Convenience functions
-async def generate_config_from_prompt(
-    prompt: str,
-    config_type: str,
-    context: Optional[Dict[str, Any]] = None
-) -> Dict[str, Any]:
-    """Generate configuration from natural language prompt."""
-    service = await get_llm_config_service()
-    
-    prompt_type_map = {
-        "crawler": ConfigurationPromptType.CRAWLER_OPTIONS,
-        "extraction": ConfigurationPromptType.EXTRACTION_SCHEMA,
-        "filter": ConfigurationPromptType.CONTENT_FILTER,
-        "search": ConfigurationPromptType.SEARCH_STRATEGY
-    }
-    
-    if config_type not in prompt_type_map:
-        raise ValueError(f"Unknown config type: {config_type}")
-    
-    request = LLMConfigurationRequest(
-        prompt=prompt,
-        prompt_type=prompt_type_map[config_type],
-        context=context
-    )
-    
-    response = await service.generate_config(request)
-    
-    if response.success:
-        return response.config
-    else:
-        raise ValueError(f"Failed to generate configuration: {response.error}")
diff --git a/apps/backend/app/services/markdown_generation.py b/apps/backend/app/services/markdown_generation.py
deleted file mode 100644
index a45165c..0000000
--- a/apps/backend/app/services/markdown_generation.py
+++ /dev/null
@@ -1,665 +0,0 @@
-"""
-Advanced markdown generation with citations and link analysis inspired by crawl4ai.
-
-This module implements sophisticated markdown generation capabilities:
-- Enhanced HTML to markdown conversion
-- Citation management and link analysis
-- Multiple output formats (raw, fit, with references)
-- Link prioritization and scoring
-"""
-
-import re
-import html
-from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Tuple, Any, Set
-from urllib.parse import urljoin, urlparse
-from dataclasses import dataclass
-
-from bs4 import BeautifulSoup, Tag, NavigableString
-import structlog
-
-from app.utils.text_processing import sanitize_text
-from app.services.content_filters import RelevantContentFilter
-
-logger = structlog.get_logger(__name__)
-
-# Pre-compile regex patterns for performance
-LINK_PATTERN = re.compile(r'!?\[([^\]]+)\]\(([^)]+?)(?:\s+"([^"]*)")?\)')
-CITATION_PATTERN = re.compile(r'\[(\d+)\]')
-WHITESPACE_PATTERN = re.compile(r'\s+')
-EMPTY_LINE_PATTERN = re.compile(r'\n\s*\n')
-
-
-@dataclass
-class MarkdownGenerationResult:
-    """Result of markdown generation process."""
-    raw_markdown: str
-    fit_markdown: Optional[str] = None
-    fit_html: Optional[str] = None  
-    references_markdown: Optional[str] = None
-    citation_map: Dict[str, int] = None
-    link_analysis: Dict[str, Any] = None
-    generation_metadata: Dict[str, Any] = None
-
-
-@dataclass
-class LinkInfo:
-    """Information about a link found during processing."""
-    url: str
-    title: str
-    text: str
-    domain: str
-    is_external: bool
-    relevance_score: float = 0.0
-    frequency: int = 1
-
-
-def fast_urljoin(base: str, url: str) -> str:
-    """Fast URL joining for common cases."""
-    if not url:
-        return base
-    if url.startswith(("http://", "https://", "mailto:", "//")):
-        return url
-    if url.startswith("/"):
-        # Handle absolute paths
-        parsed_base = urlparse(base)
-        return f"{parsed_base.scheme}://{parsed_base.netloc}{url}"
-    return urljoin(base, url)
-
-
-class MarkdownGenerationStrategy(ABC):
-    """Abstract base class for markdown generation strategies."""
-
-    def __init__(
-        self,
-        content_filter: Optional[RelevantContentFilter] = None,
-        options: Optional[Dict[str, Any]] = None,
-        verbose: bool = False,
-        content_source: str = "cleaned_html",
-    ):
-        """
-        Initialize markdown generation strategy.
-        
-        Args:
-            content_filter: Optional content filter for generating fit markdown
-            options: Additional options for markdown generation
-            verbose: Enable verbose logging
-            content_source: Source content type ("cleaned_html", "raw_html", "fit_html")
-        """
-        self.content_filter = content_filter
-        self.options = options or {}
-        self.verbose = verbose
-        self.content_source = content_source
-
-    @abstractmethod
-    async def generate_markdown(
-        self,
-        input_html: str,
-        base_url: str = "",
-        citations: bool = True,
-        **kwargs,
-    ) -> MarkdownGenerationResult:
-        """Generate markdown from the selected input HTML."""
-        pass
-
-
-class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
-    """
-    Default implementation of markdown generation strategy.
-
-    This generator:
-    1. Generates raw markdown from cleaned HTML
-    2. Converts links to citations with reference management
-    3. Generates fit markdown if content filter is provided
-    4. Performs link analysis and scoring
-    5. Returns comprehensive MarkdownGenerationResult
-    """
-
-    def __init__(
-        self,
-        content_filter: Optional[RelevantContentFilter] = None,
-        options: Optional[Dict[str, Any]] = None,
-        content_source: str = "cleaned_html",
-        **kwargs
-    ):
-        """Initialize default markdown generator."""
-        super().__init__(
-            content_filter=content_filter,
-            options=options,
-            verbose=kwargs.get("verbose", False),
-            content_source=content_source
-        )
-        
-        # Configuration options
-        self.include_images = self.options.get("include_images", True)
-        self.include_links = self.options.get("include_links", True)
-        self.include_tables = self.options.get("include_tables", True)
-        self.include_code = self.options.get("include_code", True)
-        self.max_image_width = self.options.get("max_image_width", 800)
-        self.link_preview = self.options.get("link_preview", False)
-
-    async def generate_markdown(
-        self,
-        input_html: str,
-        base_url: str = "",
-        citations: bool = True,
-        **kwargs,
-    ) -> MarkdownGenerationResult:
-        """Generate comprehensive markdown with citations and analysis."""
-        try:
-            # Parse HTML
-            soup = BeautifulSoup(input_html, 'lxml')
-            
-            # Clean and prepare HTML
-            cleaned_soup = self._clean_html(soup)
-            
-            # Generate raw markdown
-            raw_markdown = self._html_to_markdown(cleaned_soup, base_url)
-            
-            # Handle citations and link analysis
-            citation_map = {}
-            references_markdown = ""
-            link_analysis = {}
-            
-            if citations and self.include_links:
-                raw_markdown, references_markdown, citation_map, link_analysis = await self._process_citations_and_links(
-                    raw_markdown, base_url
-                )
-            
-            # Generate fit markdown if content filter is provided
-            fit_markdown = None
-            fit_html = None
-            
-            if self.content_filter:
-                filter_result = await self.content_filter.filter(input_html)
-                if filter_result.filtered_content != input_html:
-                    fit_soup = BeautifulSoup(filter_result.filtered_content, 'lxml')
-                    fit_html = str(fit_soup)
-                    fit_markdown = self._html_to_markdown(fit_soup, base_url)
-                    
-                    if citations and self.include_links:
-                        fit_markdown, _, _, _ = await self._process_citations_and_links(
-                            fit_markdown, base_url
-                        )
-            
-            # Prepare generation metadata
-            generation_metadata = {
-                "generator": self.__class__.__name__,
-                "content_source": self.content_source,
-                "citations_enabled": citations,
-                "links_processed": len(citation_map),
-                "base_url": base_url,
-                "options": self.options,
-                "filter_applied": self.content_filter is not None,
-                "raw_length": len(raw_markdown),
-                "fit_length": len(fit_markdown) if fit_markdown else 0
-            }
-            
-            return MarkdownGenerationResult(
-                raw_markdown=raw_markdown,
-                fit_markdown=fit_markdown,
-                fit_html=fit_html,
-                references_markdown=references_markdown,
-                citation_map=citation_map,
-                link_analysis=link_analysis,
-                generation_metadata=generation_metadata
-            )
-            
-        except Exception as e:
-            logger.error("markdown_generation_failed", error=str(e), base_url=base_url)
-            
-            # Return basic markdown on error
-            return MarkdownGenerationResult(
-                raw_markdown=self._fallback_markdown(input_html),
-                generation_metadata={"error": str(e)}
-            )
-
-    def _clean_html(self, soup: BeautifulSoup) -> BeautifulSoup:
-        """Clean and prepare HTML for markdown conversion."""
-        # Remove unwanted elements
-        unwanted_tags = ['script', 'style', 'noscript', 'iframe', 'embed', 'object']
-        for tag in unwanted_tags:
-            for element in soup.find_all(tag):
-                element.decompose()
-
-        # Remove comments
-        from bs4 import Comment
-        for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
-            comment.extract()
-
-        # Clean empty elements
-        for element in soup.find_all():
-            if not element.get_text(strip=True) and not element.find('img'):
-                element.decompose()
-
-        # Normalize whitespace in text nodes
-        for text_node in soup.find_all(string=True):
-            if text_node.parent.name not in ['pre', 'code']:
-                cleaned_text = WHITESPACE_PATTERN.sub(' ', text_node.strip())
-                text_node.replace_with(cleaned_text)
-
-        return soup
-
-    def _html_to_markdown(self, soup: BeautifulSoup, base_url: str) -> str:
-        """Convert HTML to markdown with proper formatting."""
-        markdown_parts = []
-        
-        # Process top-level elements
-        for element in soup.body.children if soup.body else soup.children:
-            if isinstance(element, Tag):
-                md_content = self._process_element(element, base_url)
-                if md_content:
-                    markdown_parts.append(md_content)
-            elif isinstance(element, NavigableString):
-                text = str(element).strip()
-                if text:
-                    markdown_parts.append(text)
-
-        # Join and clean up markdown
-        markdown = '\n\n'.join(markdown_parts)
-        markdown = EMPTY_LINE_PATTERN.sub('\n\n', markdown)
-        
-        return markdown.strip()
-
-    def _process_element(self, element: Tag, base_url: str) -> str:
-        """Process individual HTML elements to markdown."""
-        tag_name = element.name.lower()
-        
-        # Headers
-        if tag_name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
-            level = int(tag_name[1])
-            text = self._get_text_content(element)
-            return f"{'#' * level} {text}"
-        
-        # Paragraphs
-        elif tag_name == 'p':
-            return self._process_paragraph(element, base_url)
-        
-        # Lists
-        elif tag_name in ['ul', 'ol']:
-            return self._process_list(element, base_url, ordered=(tag_name == 'ol'))
-        
-        # Tables
-        elif tag_name == 'table' and self.include_tables:
-            return self._process_table(element, base_url)
-        
-        # Code blocks
-        elif tag_name in ['pre', 'code'] and self.include_code:
-            return self._process_code(element)
-        
-        # Blockquotes
-        elif tag_name == 'blockquote':
-            return self._process_blockquote(element, base_url)
-        
-        # Images
-        elif tag_name == 'img' and self.include_images:
-            return self._process_image(element, base_url)
-        
-        # Links
-        elif tag_name == 'a' and self.include_links:
-            return self._process_link(element, base_url)
-        
-        # Div and section elements - process children
-        elif tag_name in ['div', 'section', 'article', 'main']:
-            return self._process_container(element, base_url)
-        
-        # Inline formatting
-        elif tag_name in ['strong', 'b']:
-            text = self._get_text_content(element)
-            return f"**{text}**"
-        
-        elif tag_name in ['em', 'i']:
-            text = self._get_text_content(element)
-            return f"*{text}*"
-        
-        elif tag_name == 'mark':
-            text = self._get_text_content(element)
-            return f"=={text}=="
-        
-        # Line breaks
-        elif tag_name == 'br':
-            return '\n'
-        
-        # Horizontal rules
-        elif tag_name == 'hr':
-            return '---'
-        
-        # Default - get text content
-        else:
-            return self._get_text_content(element)
-
-    def _process_paragraph(self, element: Tag, base_url: str) -> str:
-        """Process paragraph elements with inline formatting."""
-        parts = []
-        
-        for child in element.children:
-            if isinstance(child, NavigableString):
-                text = str(child).strip()
-                if text:
-                    parts.append(text)
-            elif isinstance(child, Tag):
-                child_md = self._process_element(child, base_url)
-                if child_md:
-                    parts.append(child_md)
-        
-        return ' '.join(parts)
-
-    def _process_list(self, element: Tag, base_url: str, ordered: bool = False) -> str:
-        """Process ordered and unordered lists."""
-        items = []
-        
-        for i, li in enumerate(element.find_all('li', recursive=False)):
-            prefix = f"{i+1}. " if ordered else "- "
-            item_content = self._process_container(li, base_url)
-            
-            # Handle nested lists
-            if item_content:
-                # Indent nested content
-                lines = item_content.split('\n')
-                indented_lines = [lines[0]] + ['  ' + line for line in lines[1:]]
-                items.append(prefix + '\n'.join(indented_lines))
-        
-        return '\n'.join(items)
-
-    def _process_table(self, element: Tag, base_url: str) -> str:
-        """Process HTML tables to markdown format."""
-        rows = []
-        
-        # Process header row
-        thead = element.find('thead')
-        if thead:
-            header_row = thead.find('tr')
-            if header_row:
-                headers = []
-                for th in header_row.find_all(['th', 'td']):
-                    headers.append(self._get_text_content(th))
-                
-                if headers:
-                    rows.append('| ' + ' | '.join(headers) + ' |')
-                    rows.append('| ' + ' | '.join(['---'] * len(headers)) + ' |')
-        
-        # Process body rows
-        tbody = element.find('tbody') or element
-        for tr in tbody.find_all('tr'):
-            cells = []
-            for td in tr.find_all(['td', 'th']):
-                cell_content = self._get_text_content(td)
-                # Escape pipe characters in cell content
-                cell_content = cell_content.replace('|', '\\|')
-                cells.append(cell_content)
-            
-            if cells:
-                rows.append('| ' + ' | '.join(cells) + ' |')
-        
-        return '\n'.join(rows)
-
-    def _process_code(self, element: Tag) -> str:
-        """Process code elements."""
-        content = element.get_text()
-        
-        # Detect language from class attribute
-        language = ''
-        if element.get('class'):
-            for cls in element.get('class'):
-                if cls.startswith('language-'):
-                    language = cls[9:]
-                    break
-                elif cls.startswith('lang-'):
-                    language = cls[5:]
-                    break
-
-        if element.name == 'pre':
-            # Code block
-            return f"```{language}\n{content}\n```"
-        else:
-            # Inline code
-            return f"`{content}`"
-
-    def _process_blockquote(self, element: Tag, base_url: str) -> str:
-        """Process blockquote elements."""
-        content = self._process_container(element, base_url)
-        lines = content.split('\n')
-        quoted_lines = ['> ' + line for line in lines]
-        return '\n'.join(quoted_lines)
-
-    def _process_image(self, element: Tag, base_url: str) -> str:
-        """Process image elements."""
-        src = element.get('src', '')
-        alt = element.get('alt', '')
-        title = element.get('title', '')
-        
-        if src:
-            # Make URL absolute
-            abs_src = fast_urljoin(base_url, src)
-            
-            # Format markdown image
-            if title:
-                return f'![{alt}]({abs_src} "{title}")'
-            else:
-                return f'![{alt}]({abs_src})'
-        
-        return f'![{alt}]' if alt else ''
-
-    def _process_link(self, element: Tag, base_url: str) -> str:
-        """Process link elements."""
-        href = element.get('href', '')
-        text = self._get_text_content(element)
-        title = element.get('title', '')
-        
-        if href:
-            # Make URL absolute
-            abs_href = fast_urljoin(base_url, href)
-            
-            # Format markdown link
-            if title:
-                return f'[{text}]({abs_href} "{title}")'
-            else:
-                return f'[{text}]({abs_href})'
-        
-        return text
-
-    def _process_container(self, element: Tag, base_url: str) -> str:
-        """Process container elements by processing their children."""
-        parts = []
-        
-        for child in element.children:
-            if isinstance(child, NavigableString):
-                text = str(child).strip()
-                if text:
-                    parts.append(text)
-            elif isinstance(child, Tag):
-                child_md = self._process_element(child, base_url)
-                if child_md:
-                    parts.append(child_md)
-        
-        return '\n'.join(parts)
-
-    def _get_text_content(self, element: Tag) -> str:
-        """Get clean text content from element."""
-        text = element.get_text()
-        return sanitize_text(text)
-
-    async def _process_citations_and_links(
-        self, 
-        markdown: str, 
-        base_url: str
-    ) -> Tuple[str, str, Dict[str, int], Dict[str, Any]]:
-        """Process citations and perform link analysis."""
-        citation_map = {}
-        link_info = {}
-        citation_counter = 1
-        
-        def replace_link(match):
-            nonlocal citation_counter
-            
-            text = match.group(1)
-            url = match.group(2)
-            title = match.group(3) or ''
-            
-            # Make URL absolute
-            abs_url = fast_urljoin(base_url, url)
-            
-            # Skip if already processed
-            if abs_url in citation_map:
-                return f"{text}[{citation_map[abs_url]}]"
-            
-            # Add to citation map
-            citation_map[abs_url] = citation_counter
-            
-            # Store link information
-            link_info[abs_url] = LinkInfo(
-                url=abs_url,
-                title=title,
-                text=text,
-                domain=urlparse(abs_url).netloc,
-                is_external=urlparse(abs_url).netloc != urlparse(base_url).netloc
-            )
-            
-            result = f"{text}[{citation_counter}]"
-            citation_counter += 1
-            return result
-        
-        # Replace links with citations
-        markdown_with_citations = LINK_PATTERN.sub(replace_link, markdown)
-        
-        # Generate references markdown
-        references_lines = ["## References"]
-        for url, citation_num in sorted(citation_map.items(), key=lambda x: x[1]):
-            link_data = link_info[url]
-            title = link_data.title or link_data.text or "Link"
-            references_lines.append(f"{citation_num}. [{title}]({url})")
-        
-        references_markdown = '\n'.join(references_lines) if len(references_lines) > 1 else ""
-        
-        # Perform link analysis
-        link_analysis = await self._analyze_links(link_info, base_url)
-        
-        return markdown_with_citations, references_markdown, citation_map, link_analysis
-
-    async def _analyze_links(
-        self, 
-        link_info: Dict[str, LinkInfo], 
-        base_url: str
-    ) -> Dict[str, Any]:
-        """Analyze links for quality and relevance scoring."""
-        if not link_info:
-            return {}
-        
-        # Count domains
-        domain_counts = {}
-        external_links = 0
-        internal_links = 0
-        
-        for link_data in link_info.values():
-            domain = link_data.domain
-            domain_counts[domain] = domain_counts.get(domain, 0) + 1
-            
-            if link_data.is_external:
-                external_links += 1
-            else:
-                internal_links += 1
-        
-        # Calculate link quality scores (simplified)
-        base_domain = urlparse(base_url).netloc
-        
-        for url, link_data in link_info.items():
-            score = 0.5  # Base score
-            
-            # Authority domains get higher scores
-            authority_domains = [
-                'wikipedia.org', 'github.com', 'stackoverflow.com',
-                'mozilla.org', 'w3.org', 'ietf.org'
-            ]
-            
-            if any(domain in link_data.domain for domain in authority_domains):
-                score += 0.3
-            
-            # Internal links get slight boost for context
-            if not link_data.is_external:
-                score += 0.1
-            
-            # Links with descriptive text get higher scores
-            if len(link_data.text) > 10 and not link_data.text.startswith('http'):
-                score += 0.2
-            
-            link_data.relevance_score = min(1.0, score)
-        
-        return {
-            "total_links": len(link_info),
-            "external_links": external_links,
-            "internal_links": internal_links,
-            "unique_domains": len(domain_counts),
-            "top_domains": sorted(
-                domain_counts.items(), 
-                key=lambda x: x[1], 
-                reverse=True
-            )[:5],
-            "avg_relevance_score": sum(
-                link.relevance_score for link in link_info.values()
-            ) / len(link_info),
-            "base_domain": urlparse(base_url).netloc
-        }
-
-    def _fallback_markdown(self, html_content: str) -> str:
-        """Generate basic markdown as fallback when main conversion fails."""
-        try:
-            soup = BeautifulSoup(html_content, 'lxml')
-            
-            # Remove unwanted elements
-            for element in soup(['script', 'style', 'noscript']):
-                element.decompose()
-            
-            # Get plain text
-            text = soup.get_text()
-            return sanitize_text(text)
-            
-        except Exception:
-            return "Error: Could not convert HTML to markdown"
-
-
-# Convenience functions
-async def generate_markdown(
-    html_content: str,
-    base_url: str = "",
-    content_filter: Optional[RelevantContentFilter] = None,
-    options: Optional[Dict[str, Any]] = None,
-    citations: bool = True
-) -> MarkdownGenerationResult:
-    """
-    Generate markdown using the default generator.
-    
-    Args:
-        html_content: HTML content to convert
-        base_url: Base URL for link resolution
-        content_filter: Optional content filter for fit markdown
-        options: Additional generation options
-        citations: Whether to generate citations
-        
-    Returns:
-        MarkdownGenerationResult with all generated content
-    """
-    generator = DefaultMarkdownGenerator(
-        content_filter=content_filter,
-        options=options or {}
-    )
-    
-    return await generator.generate_markdown(
-        html_content,
-        base_url=base_url,
-        citations=citations
-    )
-
-
-async def generate_simple_markdown(html_content: str, base_url: str = "") -> str:
-    """Generate simple markdown without advanced features."""
-    generator = DefaultMarkdownGenerator(
-        options={"include_links": False, "include_images": False}
-    )
-    
-    result = await generator.generate_markdown(
-        html_content,
-        base_url=base_url,
-        citations=False
-    )
-    
-    return result.raw_markdown
diff --git a/apps/backend/app/services/multi_engine_scraper.py b/apps/backend/app/services/multi_engine_scraper.py
deleted file mode 100644
index 5f48928..0000000
--- a/apps/backend/app/services/multi_engine_scraper.py
+++ /dev/null
@@ -1,719 +0,0 @@
-"""
-Multi-engine scraping architecture inspired by Firecrawl.
-
-Provides sophisticated engine selection and fallback capabilities for various content types.
-"""
-
-import asyncio
-import time
-from typing import Dict, List, Optional, Any, Union, Tuple
-from enum import Enum
-from dataclasses import dataclass, field
-from abc import ABC, abstractmethod
-import httpx
-import structlog
-
-from app.config import get_settings
-from app.models.responses import ScrapedContent, ContentMetadata
-from app.models.requests import ScrapingConfig
-from app.services.scraping import ContentScrapingService
-from app.utils.text_processing import sanitize_text, detect_language, calculate_text_quality
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class EngineType(Enum):
-    """Available scraping engines."""
-    INDEX = "index"  # Pre-indexed content
-    INDEX_DOCUMENTS = "index;documents"  # Pre-indexed documents
-    FIRE_ENGINE_CDP = "fire-engine;chrome-cdp"
-    FIRE_ENGINE_CDP_STEALTH = "fire-engine;chrome-cdp;stealth"
-    FIRE_ENGINE_CDP_RETRY = "fire-engine(retry);chrome-cdp"  
-    FIRE_ENGINE_CDP_RETRY_STEALTH = "fire-engine(retry);chrome-cdp;stealth"
-    FIRE_ENGINE_PLAYWRIGHT = "fire-engine;playwright"
-    FIRE_ENGINE_PLAYWRIGHT_STEALTH = "fire-engine;playwright;stealth"
-    FIRE_ENGINE_TLSCLIENT = "fire-engine;tlsclient"
-    FIRE_ENGINE_TLSCLIENT_STEALTH = "fire-engine;tlsclient;stealth"
-    PLAYWRIGHT = "playwright"
-    FETCH = "fetch"
-    PDF = "pdf"
-    DOCX = "docx"
-
-
-@dataclass
-class EngineCapabilities:
-    """Capabilities supported by each engine."""
-    actions: bool = False
-    wait_for: bool = False
-    screenshot: bool = False
-    screenshot_full: bool = False
-    pdf: bool = False
-    docx: bool = False
-    atsv: bool = False  # Accessibility tree structured view
-    mobile: bool = False
-    location: bool = False
-    skip_tls_verification: bool = False
-    use_fast_mode: bool = False
-    stealth_proxy: bool = False
-    disable_adblock: bool = False
-
-
-@dataclass  
-class EngineConfig:
-    """Configuration for scraping engine."""
-    engine_type: EngineType
-    capabilities: EngineCapabilities
-    quality: int  # Higher = preferred, negative = specialty
-    max_reasonable_time: int  # Maximum reasonable processing time (ms)
-    enabled: bool = True
-
-
-@dataclass
-class ScrapeRequest:
-    """Enhanced scrape request with engine selection."""
-    url: str
-    config: ScrapingConfig
-    preferred_engine: Optional[EngineType] = None
-    required_capabilities: List[str] = field(default_factory=list)
-    timeout: int = 30
-    retries: int = 2
-
-
-@dataclass
-class ScrapeResult:
-    """Enhanced scrape result with engine metadata."""
-    content: ScrapedContent
-    engine_used: EngineType
-    processing_time: float
-    attempts: int
-    success: bool
-    error: Optional[str] = None
-
-
-class BaseEngine(ABC):
-    """Abstract base class for scraping engines."""
-    
-    def __init__(self, engine_type: EngineType, config: EngineConfig):
-        self.engine_type = engine_type
-        self.config = config
-        self.stats = {"requests": 0, "successes": 0, "failures": 0, "avg_time": 0.0}
-    
-    @abstractmethod
-    async def scrape(self, request: ScrapeRequest) -> ScrapedContent:
-        """Scrape content using this engine."""
-        pass
-    
-    def can_handle(self, request: ScrapeRequest) -> bool:
-        """Check if engine can handle the request."""
-        # Check required capabilities
-        for capability in request.required_capabilities:
-            if not getattr(self.config.capabilities, capability, False):
-                return False
-        return True
-    
-    def update_stats(self, success: bool, processing_time: float):
-        """Update engine statistics."""
-        self.stats["requests"] += 1
-        if success:
-            self.stats["successes"] += 1
-        else:
-            self.stats["failures"] += 1
-        
-        # Update average time
-        if self.stats["requests"] > 0:
-            self.stats["avg_time"] = (
-                (self.stats["avg_time"] * (self.stats["requests"] - 1) + processing_time) 
-                / self.stats["requests"]
-            )
-
-
-class IndexEngine(BaseEngine):
-    """Index-based scraping for pre-cached content."""
-    
-    def __init__(self):
-        super().__init__(
-            EngineType.INDEX,
-            EngineConfig(
-                engine_type=EngineType.INDEX,
-                capabilities=EngineCapabilities(
-                    wait_for=True,
-                    screenshot=True, 
-                    screenshot_full=True,
-                    mobile=True,
-                    location=True,
-                    skip_tls_verification=True,
-                    use_fast_mode=True
-                ),
-                quality=1000,  # Highest priority
-                max_reasonable_time=2000
-            )
-        )
-        self.client = httpx.AsyncClient(timeout=30)
-    
-    async def scrape(self, request: ScrapeRequest) -> ScrapedContent:
-        """Scrape using index/cache."""
-        start_time = time.time()
-        
-        try:
-            # Check if we have cached content
-            cached_content = await self._get_cached_content(request.url)
-            if cached_content:
-                processing_time = time.time() - start_time
-                self.update_stats(True, processing_time)
-                return cached_content
-            
-            # If not cached, fall back to fast scraping
-            return await self._fast_scrape(request)
-            
-        except Exception as e:
-            processing_time = time.time() - start_time
-            self.update_stats(False, processing_time)
-            raise e
-    
-    async def _get_cached_content(self, url: str) -> Optional[ScrapedContent]:
-        """Check for cached content."""
-        # Implementation would check Redis/database cache
-        # For now, return None to indicate no cache
-        return None
-    
-    async def _fast_scrape(self, request: ScrapeRequest) -> ScrapedContent:
-        """Perform fast scraping."""
-        response = await self.client.get(request.url)
-        response.raise_for_status()
-        
-        # Basic content extraction
-        from bs4 import BeautifulSoup
-        soup = BeautifulSoup(response.text, 'lxml')
-        
-        # Clean content
-        for element in soup(['script', 'style', 'noscript']):
-            element.decompose()
-        
-        text = sanitize_text(soup.get_text())
-        title = soup.find('title').get_text() if soup.find('title') else ""
-        
-        return ScrapedContent(
-            url=request.url,
-            title=title,
-            text=text,
-            html=response.text,
-            extraction_success=True,
-            word_count=len(text.split()) if text else 0,
-            language_detected=detect_language(text),
-            content_quality_score=calculate_text_quality(text),
-            metadata=ContentMetadata()
-        )
-
-
-class FireEngineEngine(BaseEngine):
-    """Fire Engine based scraping with multiple protocols."""
-    
-    def __init__(self, engine_type: EngineType):
-        # Configure capabilities based on engine variant
-        if "stealth" in engine_type.value:
-            capabilities = EngineCapabilities(
-                actions=True,
-                wait_for=True,
-                screenshot=True,
-                screenshot_full=True,
-                location=True,
-                mobile=True,
-                skip_tls_verification=True,
-                stealth_proxy=True
-            )
-            quality = -2 if "retry" in engine_type.value else -1
-        else:
-            capabilities = EngineCapabilities(
-                actions=True,
-                wait_for=True,
-                screenshot=True,
-                screenshot_full=True,
-                location=True,
-                mobile=True,
-                skip_tls_verification=True
-            )
-            quality = 45 if "retry" in engine_type.value else 50
-        
-        super().__init__(
-            engine_type,
-            EngineConfig(
-                engine_type=engine_type,
-                capabilities=capabilities,
-                quality=quality,
-                max_reasonable_time=60000,
-                enabled=bool(getattr(settings, 'fire_engine_url', None))
-            )
-        )
-        self.client = httpx.AsyncClient(timeout=120)
-    
-    async def scrape(self, request: ScrapeRequest) -> ScrapedContent:
-        """Scrape using Fire Engine."""
-        start_time = time.time()
-        
-        try:
-            # Prepare Fire Engine request
-            payload = {
-                "url": request.url,
-                "options": {
-                    "engine": self._get_engine_protocol(),
-                    "timeout": request.timeout * 1000,  # Convert to ms
-                    "waitFor": getattr(request.config, 'wait_time', 0) * 1000,
-                }
-            }
-            
-            # Add stealth options if needed
-            if "stealth" in self.engine_type.value:
-                payload["options"]["stealth"] = True
-                payload["options"]["antiDetection"] = True
-            
-            # Add mobile options if requested
-            if getattr(request.config, 'mobile_mode', False):
-                payload["options"]["mobile"] = True
-            
-            # Add screenshot if needed
-            if getattr(request.config, 'screenshot', False):
-                payload["options"]["screenshot"] = True
-            
-            fire_engine_url = getattr(settings, 'fire_engine_url', '')
-            response = await self.client.post(
-                f"{fire_engine_url}/scrape",
-                json=payload,
-                headers={"Content-Type": "application/json"}
-            )
-            response.raise_for_status()
-            
-            data = response.json()
-            scraped_content = self._process_fire_engine_response(data, request.url)
-            
-            processing_time = time.time() - start_time
-            self.update_stats(True, processing_time)
-            return scraped_content
-            
-        except Exception as e:
-            processing_time = time.time() - start_time
-            self.update_stats(False, processing_time)
-            raise e
-    
-    def _get_engine_protocol(self) -> str:
-        """Get the engine protocol for Fire Engine."""
-        if "chrome-cdp" in self.engine_type.value:
-            return "chrome-cdp"
-        elif "playwright" in self.engine_type.value:
-            return "playwright"
-        elif "tlsclient" in self.engine_type.value:
-            return "tlsclient"
-        return "chrome-cdp"  # default
-    
-    def _process_fire_engine_response(self, data: Dict[str, Any], url: str) -> ScrapedContent:
-        """Process Fire Engine response."""
-        content = data.get("content", {})
-        html = content.get("html", "")
-        text = content.get("text", "")
-        title = content.get("title", "")
-        
-        # Extract metadata
-        metadata_dict = content.get("metadata", {})
-        metadata = ContentMetadata(
-            title=metadata_dict.get("title", title),
-            description=metadata_dict.get("description", ""),
-            author=metadata_dict.get("author"),
-            published_date=metadata_dict.get("publishedDate"),
-            keywords=metadata_dict.get("keywords", [])
-        )
-        
-        return ScrapedContent(
-            url=url,
-            title=title,
-            text=text,
-            html=html,
-            extraction_success=True,
-            word_count=len(text.split()) if text else 0,
-            language_detected=detect_language(text),
-            content_quality_score=calculate_text_quality(text),
-            metadata=metadata,
-            screenshots=content.get("screenshots", []) if content.get("screenshots") else None
-        )
-
-
-class PlaywrightEngine(BaseEngine):
-    """Playwright-based scraping engine."""
-    
-    def __init__(self):
-        super().__init__(
-            EngineType.PLAYWRIGHT,
-            EngineConfig(
-                engine_type=EngineType.PLAYWRIGHT,
-                capabilities=EngineCapabilities(
-                    wait_for=True,
-                    screenshot=True,
-                    screenshot_full=True,
-                    disable_adblock=True
-                ),
-                quality=35,
-                max_reasonable_time=45000,
-                enabled=bool(getattr(settings, 'playwright_service_url', None))
-            )
-        )
-        self.client = httpx.AsyncClient(timeout=60)
-    
-    async def scrape(self, request: ScrapeRequest) -> ScrapedContent:
-        """Scrape using Playwright service."""
-        start_time = time.time()
-        
-        try:
-            payload = {
-                "url": request.url,
-                "options": {
-                    "waitTime": getattr(request.config, 'wait_time', 0) * 1000,
-                    "timeout": request.timeout * 1000,
-                }
-            }
-            
-            if getattr(request.config, 'screenshot', False):
-                payload["options"]["screenshot"] = True
-            
-            playwright_url = getattr(settings, 'playwright_service_url', '')
-            response = await self.client.post(
-                f"{playwright_url}/scrape",
-                json=payload,
-                headers={"Content-Type": "application/json"}
-            )
-            response.raise_for_status()
-            
-            data = response.json()
-            scraped_content = self._process_playwright_response(data, request.url)
-            
-            processing_time = time.time() - start_time
-            self.update_stats(True, processing_time)
-            return scraped_content
-            
-        except Exception as e:
-            processing_time = time.time() - start_time
-            self.update_stats(False, processing_time)
-            raise e
-    
-    def _process_playwright_response(self, data: Dict[str, Any], url: str) -> ScrapedContent:
-        """Process Playwright response."""
-        html = data.get("html", "")
-        text = data.get("text", "")
-        title = data.get("title", "")
-        
-        return ScrapedContent(
-            url=url,
-            title=title,
-            text=text,
-            html=html,
-            extraction_success=True,
-            word_count=len(text.split()) if text else 0,
-            language_detected=detect_language(text),
-            content_quality_score=calculate_text_quality(text),
-            metadata=ContentMetadata(),
-            screenshots=data.get("screenshots", []) if data.get("screenshots") else None
-        )
-
-
-class FetchEngine(BaseEngine):
-    """Simple HTTP fetch engine for basic scraping."""
-    
-    def __init__(self):
-        super().__init__(
-            EngineType.FETCH,
-            EngineConfig(
-                engine_type=EngineType.FETCH,
-                capabilities=EngineCapabilities(use_fast_mode=True),
-                quality=10,  # Low quality, fallback option
-                max_reasonable_time=10000
-            )
-        )
-        self.client = httpx.AsyncClient(
-            timeout=30,
-            headers={
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-            }
-        )
-    
-    async def scrape(self, request: ScrapeRequest) -> ScrapedContent:
-        """Scrape using basic HTTP fetch."""
-        start_time = time.time()
-        
-        try:
-            response = await self.client.get(request.url)
-            response.raise_for_status()
-            
-            # Use existing scraping service for processing
-            basic_service = ContentScrapingService()
-            scraped_content = await basic_service._scrape_single_url(request.url, request.config)
-            
-            processing_time = time.time() - start_time
-            self.update_stats(True, processing_time)
-            return scraped_content
-            
-        except Exception as e:
-            processing_time = time.time() - start_time
-            self.update_stats(False, processing_time)
-            raise e
-
-
-class PDFEngine(BaseEngine):
-    """PDF document processing engine."""
-    
-    def __init__(self):
-        super().__init__(
-            EngineType.PDF,
-            EngineConfig(
-                engine_type=EngineType.PDF,
-                capabilities=EngineCapabilities(pdf=True),
-                quality=-10,  # Specialty engine
-                max_reasonable_time=30000
-            )
-        )
-        self.client = httpx.AsyncClient(timeout=60)
-    
-    async def scrape(self, request: ScrapeRequest) -> ScrapedContent:
-        """Scrape PDF content."""
-        start_time = time.time()
-        
-        try:
-            # Download PDF
-            response = await self.client.get(request.url)
-            response.raise_for_status()
-            
-            # Extract text from PDF (would need PyPDF2 or similar)
-            # For now, simplified implementation
-            text = f"PDF content from {request.url}"
-            title = request.url.split('/')[-1]
-            
-            scraped_content = ScrapedContent(
-                url=request.url,
-                title=title,
-                text=text,
-                extraction_success=True,
-                word_count=len(text.split()),
-                language_detected=detect_language(text),
-                content_quality_score=calculate_text_quality(text),
-                metadata=ContentMetadata(content_type="application/pdf")
-            )
-            
-            processing_time = time.time() - start_time
-            self.update_stats(True, processing_time)
-            return scraped_content
-            
-        except Exception as e:
-            processing_time = time.time() - start_time
-            self.update_stats(False, processing_time)
-            raise e
-
-
-class MultiEngineScrapingService:
-    """
-    Multi-engine scraping service with intelligent engine selection.
-    
-    Provides automatic engine selection based on:
-    - URL content type detection
-    - Required capabilities
-    - Engine availability and performance
-    - Fallback strategies
-    """
-    
-    def __init__(self):
-        """Initialize multi-engine scraping service."""
-        self.engines: Dict[EngineType, BaseEngine] = {}
-        self._initialize_engines()
-        self.usage_stats = {"total_requests": 0, "successful_requests": 0}
-    
-    def _initialize_engines(self):
-        """Initialize all available engines."""
-        # Index engines (highest priority)
-        self.engines[EngineType.INDEX] = IndexEngine()
-        
-        # Fire Engine variants (if available)
-        fire_engine_types = [
-            EngineType.FIRE_ENGINE_CDP,
-            EngineType.FIRE_ENGINE_CDP_STEALTH,
-            EngineType.FIRE_ENGINE_CDP_RETRY,
-            EngineType.FIRE_ENGINE_CDP_RETRY_STEALTH,
-            EngineType.FIRE_ENGINE_PLAYWRIGHT,
-            EngineType.FIRE_ENGINE_PLAYWRIGHT_STEALTH,
-            EngineType.FIRE_ENGINE_TLSCLIENT,
-            EngineType.FIRE_ENGINE_TLSCLIENT_STEALTH,
-        ]
-        
-        for engine_type in fire_engine_types:
-            engine = FireEngineEngine(engine_type)
-            if engine.config.enabled:
-                self.engines[engine_type] = engine
-        
-        # Playwright (if available)
-        playwright_engine = PlaywrightEngine()
-        if playwright_engine.config.enabled:
-            self.engines[EngineType.PLAYWRIGHT] = playwright_engine
-        
-        # Specialty engines
-        self.engines[EngineType.PDF] = PDFEngine()
-        
-        # Fetch engine (always available as fallback)
-        self.engines[EngineType.FETCH] = FetchEngine()
-        
-        logger.info("multi_engine_initialized", 
-                   engines=list(self.engines.keys()),
-                   total_engines=len(self.engines))
-    
-    async def scrape(
-        self,
-        url: str,
-        config: ScrapingConfig,
-        preferred_engine: Optional[EngineType] = None,
-        required_capabilities: List[str] = None
-    ) -> ScrapeResult:
-        """
-        Scrape URL using the best available engine.
-        
-        Args:
-            url: URL to scrape
-            config: Scraping configuration
-            preferred_engine: Preferred engine type
-            required_capabilities: Required engine capabilities
-            
-        Returns:
-            ScrapeResult with content and metadata
-        """
-        request = ScrapeRequest(
-            url=url,
-            config=config,
-            preferred_engine=preferred_engine,
-            required_capabilities=required_capabilities or []
-        )
-        
-        self.usage_stats["total_requests"] += 1
-        
-        # Select appropriate engine
-        selected_engines = await self._select_engines(request)
-        
-        if not selected_engines:
-            logger.error("no_suitable_engine_found", url=url, capabilities=required_capabilities)
-            return ScrapeResult(
-                content=ScrapedContent(url=url, extraction_success=False, text=""),
-                engine_used=EngineType.FETCH,
-                processing_time=0,
-                attempts=0,
-                success=False,
-                error="No suitable engine found"
-            )
-        
-        # Try engines in order
-        last_error = None
-        attempts = 0
-        
-        for engine_type in selected_engines:
-            engine = self.engines[engine_type]
-            attempts += 1
-            
-            try:
-                logger.info("attempting_scrape", url=url, engine=engine_type.value, attempt=attempts)
-                start_time = time.time()
-                
-                content = await engine.scrape(request)
-                processing_time = time.time() - start_time
-                
-                self.usage_stats["successful_requests"] += 1
-                
-                return ScrapeResult(
-                    content=content,
-                    engine_used=engine_type,
-                    processing_time=processing_time,
-                    attempts=attempts,
-                    success=True
-                )
-                
-            except Exception as e:
-                last_error = str(e)
-                logger.warning("engine_failed", 
-                             url=url, 
-                             engine=engine_type.value, 
-                             error=str(e))
-                continue
-        
-        # All engines failed
-        return ScrapeResult(
-            content=ScrapedContent(url=url, extraction_success=False, text=""),
-            engine_used=selected_engines[0] if selected_engines else EngineType.FETCH,
-            processing_time=0,
-            attempts=attempts,
-            success=False,
-            error=last_error or "All engines failed"
-        )
-    
-    async def _select_engines(self, request: ScrapeRequest) -> List[EngineType]:
-        """Select appropriate engines based on request requirements."""
-        suitable_engines = []
-        
-        # Check if user prefers a specific engine
-        if request.preferred_engine and request.preferred_engine in self.engines:
-            engine = self.engines[request.preferred_engine]
-            if engine.can_handle(request):
-                suitable_engines.append(request.preferred_engine)
-        
-        # Find all suitable engines
-        for engine_type, engine in self.engines.items():
-            if engine_type == request.preferred_engine:
-                continue  # Already added
-            
-            if engine.can_handle(request):
-                suitable_engines.append(engine_type)
-        
-        # Sort by quality (higher quality first)
-        suitable_engines.sort(key=lambda e: self.engines[e].config.quality, reverse=True)
-        
-        # Special handling for specific content types
-        url_lower = request.url.lower()
-        if url_lower.endswith('.pdf'):
-            # Prioritize PDF engine
-            if EngineType.PDF in suitable_engines:
-                suitable_engines.remove(EngineType.PDF)
-                suitable_engines.insert(0, EngineType.PDF)
-        
-        return suitable_engines
-    
-    async def get_engine_stats(self) -> Dict[str, Any]:
-        """Get comprehensive engine statistics."""
-        engine_stats = {}
-        for engine_type, engine in self.engines.items():
-            engine_stats[engine_type.value] = {
-                "config": {
-                    "quality": engine.config.quality,
-                    "max_reasonable_time": engine.config.max_reasonable_time,
-                    "enabled": engine.config.enabled
-                },
-                "stats": engine.stats,
-                "capabilities": {
-                    field.name: getattr(engine.config.capabilities, field.name)
-                    for field in engine.config.capabilities.__dataclass_fields__.values()
-                }
-            }
-        
-        return {
-            "engines": engine_stats,
-            "usage_stats": self.usage_stats,
-            "total_engines": len(self.engines)
-        }
-    
-    async def cleanup(self):
-        """Cleanup all engine resources."""
-        for engine in self.engines.values():
-            if hasattr(engine, 'client') and engine.client:
-                await engine.client.aclose()
-
-
-# Singleton instance
-_multi_engine_service: Optional[MultiEngineScrapingService] = None
-
-
-async def get_multi_engine_service() -> MultiEngineScrapingService:
-    """Get or create multi-engine scraping service instance."""
-    global _multi_engine_service
-    
-    if _multi_engine_service is None:
-        _multi_engine_service = MultiEngineScrapingService()
-    
-    return _multi_engine_service
diff --git a/apps/backend/app/services/multi_entity_extraction.py b/apps/backend/app/services/multi_entity_extraction.py
deleted file mode 100644
index 69ff260..0000000
--- a/apps/backend/app/services/multi_entity_extraction.py
+++ /dev/null
@@ -1,828 +0,0 @@
-"""
-Multi-entity extraction service inspired by Firecrawl's advanced extraction capabilities.
-
-Provides intelligent cross-URL data extraction and entity linking for complex data scenarios.
-"""
-
-import asyncio
-import time
-import json
-from typing import Dict, List, Optional, Any, Union, Set, Tuple
-from dataclasses import dataclass, field
-from enum import Enum
-import structlog
-from urllib.parse import urlparse, urljoin
-import re
-
-from app.config import get_settings
-from app.models.requests import ScrapingConfig
-from app.models.responses import ScrapedContent, ContentMetadata
-from app.services.enhanced_scraping import get_enhanced_scraping_service
-from app.services.llm_configuration import get_llm_config_service
-from app.utils.text_processing import sanitize_text, extract_entities, clean_tokens
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class ExtractionStrategy(Enum):
-    """Multi-entity extraction strategies."""
-    LINKED_ENTITIES = "linked_entities"  # Extract entities and find related URLs
-    HIERARCHICAL = "hierarchical"  # Follow hierarchical relationships
-    SEMANTIC_SIMILARITY = "semantic_similarity"  # Group by semantic similarity
-    TEMPORAL_SEQUENCE = "temporal_sequence"  # Time-based entity relationships
-    CROSS_REFERENCE = "cross_reference"  # Cross-reference validation
-
-
-@dataclass
-class EntityRelation:
-    """Represents relationship between entities."""
-    source_url: str
-    target_url: str
-    relation_type: str
-    confidence: float
-    evidence: List[str]
-
-
-@dataclass
-class ExtractedEntity:
-    """Represents an extracted entity with metadata."""
-    id: str
-    entity_type: str
-    value: Any
-    confidence: float
-    source_url: str
-    extraction_method: str
-    context: Optional[str] = None
-    attributes: Dict[str, Any] = field(default_factory=dict)
-    related_entities: List[str] = field(default_factory=list)
-
-
-@dataclass
-class MultiEntityExtractionRequest:
-    """Request for multi-entity extraction."""
-    urls: List[str]
-    schema: Dict[str, Any]
-    extraction_strategy: ExtractionStrategy = ExtractionStrategy.LINKED_ENTITIES
-    max_related_urls: int = 50
-    similarity_threshold: float = 0.7
-    cross_validate: bool = True
-    follow_links: bool = True
-    max_depth: int = 2
-    include_metadata: bool = True
-    timeout: int = 300  # 5 minutes
-
-
-@dataclass
-class MultiEntityExtractionResult:
-    """Result of multi-entity extraction."""
-    request_id: str
-    entities: List[ExtractedEntity]
-    relations: List[EntityRelation]
-    discovered_urls: List[str]
-    validation_results: Dict[str, Any]
-    extraction_metadata: Dict[str, Any]
-    processing_time: float
-    success: bool
-    errors: List[str] = field(default_factory=list)
-
-
-class EntityLinker:
-    """Links entities across multiple URLs and content sources."""
-    
-    def __init__(self):
-        self.entity_cache: Dict[str, List[ExtractedEntity]] = {}
-        self.url_graph: Dict[str, Set[str]] = {}
-    
-    async def discover_related_urls(
-        self, 
-        base_urls: List[str], 
-        strategy: ExtractionStrategy,
-        max_urls: int = 50
-    ) -> List[str]:
-        """Discover URLs related to base URLs using various strategies."""
-        
-        discovered = set(base_urls)
-        
-        if strategy == ExtractionStrategy.LINKED_ENTITIES:
-            # Find URLs through link analysis
-            for url in base_urls:
-                linked_urls = await self._find_linked_urls(url)
-                discovered.update(linked_urls[:max_urls // len(base_urls)])
-        
-        elif strategy == ExtractionStrategy.HIERARCHICAL:
-            # Find URLs through hierarchical relationships
-            for url in base_urls:
-                hierarchical_urls = await self._find_hierarchical_urls(url)
-                discovered.update(hierarchical_urls[:max_urls // len(base_urls)])
-        
-        elif strategy == ExtractionStrategy.SEMANTIC_SIMILARITY:
-            # Find URLs through content similarity
-            for url in base_urls:
-                similar_urls = await self._find_similar_urls(url)
-                discovered.update(similar_urls[:max_urls // len(base_urls)])
-        
-        return list(discovered)[:max_urls]
-    
-    async def _find_linked_urls(self, base_url: str) -> List[str]:
-        """Find URLs through link analysis."""
-        try:
-            # Scrape base URL to extract links
-            scraping_service = await get_enhanced_scraping_service()
-            config = ScrapingConfig(
-                urls=[base_url],
-                extract_links=True,
-                extract_text=True
-            )
-            
-            results = await scraping_service.scrape_urls_enhanced([base_url], config)
-            if not results or not results[0].extraction_success:
-                return []
-            
-            links = results[0].links or []
-            
-            # Filter and score links based on relevance
-            relevant_links = []
-            base_domain = urlparse(base_url).netloc
-            
-            for link in links:
-                # Prefer internal links but allow some external
-                link_domain = urlparse(link.url).netloc
-                if link_domain == base_domain:
-                    relevant_links.append(link.url)
-                elif len(relevant_links) < 10:  # Limited external links
-                    relevant_links.append(link.url)
-            
-            return relevant_links[:20]
-            
-        except Exception as e:
-            logger.warning("failed_to_find_linked_urls", url=base_url, error=str(e))
-            return []
-    
-    async def _find_hierarchical_urls(self, base_url: str) -> List[str]:
-        """Find URLs through hierarchical relationships."""
-        hierarchical_urls = []
-        parsed = urlparse(base_url)
-        
-        # Generate parent URLs
-        path_parts = parsed.path.strip('/').split('/')
-        for i in range(len(path_parts)):
-            parent_path = '/'.join(path_parts[:i+1])
-            if parent_path and parent_path != parsed.path.strip('/'):
-                parent_url = f"{parsed.scheme}://{parsed.netloc}/{parent_path}"
-                hierarchical_urls.append(parent_url)
-        
-        # Generate sibling URLs (same level)
-        if len(path_parts) > 1:
-            parent_path = '/'.join(path_parts[:-1])
-            # This would typically query a sitemap or use discovery patterns
-            # For now, we'll generate common patterns
-            common_patterns = ['index.html', 'about.html', 'contact.html', 'services.html']
-            for pattern in common_patterns:
-                sibling_url = f"{parsed.scheme}://{parsed.netloc}/{parent_path}/{pattern}"
-                hierarchical_urls.append(sibling_url)
-        
-        return hierarchical_urls
-    
-    async def _find_similar_urls(self, base_url: str) -> List[str]:
-        """Find URLs through content similarity."""
-        # This would typically use semantic search or content analysis
-        # For now, return pattern-based URLs
-        similar_urls = []
-        parsed = urlparse(base_url)
-        
-        # Generate pattern-based similar URLs
-        if 'blog' in base_url:
-            # Find other blog posts
-            base_blog_url = base_url.split('/blog/')[0] + '/blog/'
-            for i in range(1, 6):
-                similar_urls.append(f"{base_blog_url}post-{i}")
-        
-        if 'product' in base_url:
-            # Find other products
-            base_product_url = base_url.split('/product/')[0] + '/product/'
-            for i in range(1, 6):
-                similar_urls.append(f"{base_product_url}item-{i}")
-        
-        return similar_urls
-
-
-class EntityExtractor:
-    """Extracts structured entities from content using various methods."""
-    
-    def __init__(self):
-        self.extraction_cache: Dict[str, List[ExtractedEntity]] = {}
-    
-    async def extract_entities(
-        self,
-        content: ScrapedContent,
-        schema: Dict[str, Any],
-        extraction_method: str = "llm"
-    ) -> List[ExtractedEntity]:
-        """Extract entities from scraped content based on schema."""
-        
-        # Check cache first
-        cache_key = f"{content.url}:{hash(json.dumps(schema, sort_keys=True))}"
-        if cache_key in self.extraction_cache:
-            return self.extraction_cache[cache_key]
-        
-        entities = []
-        
-        try:
-            if extraction_method == "llm":
-                entities = await self._extract_with_llm(content, schema)
-            elif extraction_method == "regex":
-                entities = await self._extract_with_regex(content, schema)
-            elif extraction_method == "css":
-                entities = await self._extract_with_css(content, schema)
-            else:
-                entities = await self._extract_with_llm(content, schema)  # Default to LLM
-            
-            # Cache results
-            self.extraction_cache[cache_key] = entities
-            
-        except Exception as e:
-            logger.error("entity_extraction_failed", 
-                        url=content.url, 
-                        method=extraction_method, 
-                        error=str(e))
-        
-        return entities
-    
-    async def _extract_with_llm(
-        self, 
-        content: ScrapedContent, 
-        schema: Dict[str, Any]
-    ) -> List[ExtractedEntity]:
-        """Extract entities using LLM."""
-        
-        try:
-            llm_service = await get_llm_config_service()
-            
-            # Prepare extraction prompt
-            prompt = f"""
-            Extract structured data from the following web content according to the schema.
-            
-            Schema: {json.dumps(schema, indent=2)}
-            
-            Content Title: {content.title}
-            Content Text: {content.text[:5000]}...
-            
-            Extract all relevant entities that match the schema. For each entity, provide:
-            1. The extracted value
-            2. Confidence score (0.0-1.0)
-            3. Context/evidence from the content
-            
-            Return as JSON array of entities.
-            """
-            
-            from app.services.llm_configuration import LLMConfigurationRequest, ConfigurationPromptType
-            
-            request = LLMConfigurationRequest(
-                prompt=prompt,
-                prompt_type=ConfigurationPromptType.EXTRACTION_SCHEMA
-            )
-            
-            response = await llm_service.generate_config(request)
-            
-            if response.success and response.config:
-                entities = []
-                extracted_data = response.config.get("extracted_entities", [])
-                
-                for i, data in enumerate(extracted_data):
-                    entity = ExtractedEntity(
-                        id=f"{content.url}:llm:{i}",
-                        entity_type=data.get("type", "unknown"),
-                        value=data.get("value"),
-                        confidence=data.get("confidence", 0.5),
-                        source_url=content.url,
-                        extraction_method="llm",
-                        context=data.get("context"),
-                        attributes=data.get("attributes", {})
-                    )
-                    entities.append(entity)
-                
-                return entities
-            
-        except Exception as e:
-            logger.error("llm_extraction_failed", url=content.url, error=str(e))
-        
-        return []
-    
-    async def _extract_with_regex(
-        self, 
-        content: ScrapedContent, 
-        schema: Dict[str, Any]
-    ) -> List[ExtractedEntity]:
-        """Extract entities using regex patterns."""
-        entities = []
-        
-        # Extract common patterns
-        patterns = {
-            "email": r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',
-            "phone": r'\b\d{3}[-.]?\d{3}[-.]?\d{4}\b',
-            "url": r'https?://[^\s<>"{}|\\^`\[\]]+',
-            "price": r'\$\d+(?:\.\d{2})?',
-            "date": r'\b\d{1,2}[/-]\d{1,2}[/-]\d{4}\b'
-        }
-        
-        text = content.text or ""
-        
-        for entity_type, pattern in patterns.items():
-            matches = re.finditer(pattern, text, re.IGNORECASE)
-            for i, match in enumerate(matches):
-                entity = ExtractedEntity(
-                    id=f"{content.url}:regex:{entity_type}:{i}",
-                    entity_type=entity_type,
-                    value=match.group(),
-                    confidence=0.8,  # Regex typically high confidence
-                    source_url=content.url,
-                    extraction_method="regex",
-                    context=text[max(0, match.start()-50):match.end()+50]
-                )
-                entities.append(entity)
-        
-        return entities
-    
-    async def _extract_with_css(
-        self, 
-        content: ScrapedContent, 
-        schema: Dict[str, Any]
-    ) -> List[ExtractedEntity]:
-        """Extract entities using CSS selectors."""
-        entities = []
-        
-        if not content.html:
-            return entities
-        
-        try:
-            from bs4 import BeautifulSoup
-            soup = BeautifulSoup(content.html, 'lxml')
-            
-            # Common CSS patterns for structured data
-            selectors = {
-                "title": ["h1", "h2", ".title", "#title"],
-                "price": [".price", ".cost", "[data-price]"],
-                "description": [".description", ".summary", ".excerpt"],
-                "author": [".author", ".by", "[rel='author']"],
-                "date": [".date", ".published", "time"],
-            }
-            
-            for entity_type, css_selectors in selectors.items():
-                for selector in css_selectors:
-                    elements = soup.select(selector)
-                    for i, element in enumerate(elements[:5]):  # Limit per type
-                        entity = ExtractedEntity(
-                            id=f"{content.url}:css:{entity_type}:{i}",
-                            entity_type=entity_type,
-                            value=sanitize_text(element.get_text()),
-                            confidence=0.7,
-                            source_url=content.url,
-                            extraction_method="css",
-                            attributes={"selector": selector}
-                        )
-                        entities.append(entity)
-            
-        except Exception as e:
-            logger.error("css_extraction_failed", url=content.url, error=str(e))
-        
-        return entities
-
-
-class EntityValidator:
-    """Validates extracted entities through cross-referencing and consistency checks."""
-    
-    async def validate_entities(
-        self, 
-        entities: List[ExtractedEntity],
-        validation_rules: Optional[Dict[str, Any]] = None
-    ) -> Dict[str, Any]:
-        """Validate extracted entities."""
-        
-        validation_result = {
-            "total_entities": len(entities),
-            "valid_entities": 0,
-            "invalid_entities": 0,
-            "validation_errors": [],
-            "consistency_score": 0.0,
-            "cross_reference_matches": 0
-        }
-        
-        # Group entities by type
-        entities_by_type = {}
-        for entity in entities:
-            if entity.entity_type not in entities_by_type:
-                entities_by_type[entity.entity_type] = []
-            entities_by_type[entity.entity_type].append(entity)
-        
-        # Validate each entity type
-        for entity_type, type_entities in entities_by_type.items():
-            type_validation = await self._validate_entity_type(entity_type, type_entities)
-            validation_result["valid_entities"] += type_validation["valid_count"]
-            validation_result["invalid_entities"] += type_validation["invalid_count"]
-            validation_result["validation_errors"].extend(type_validation["errors"])
-        
-        # Cross-reference validation
-        cross_ref_score = await self._cross_reference_validation(entities_by_type)
-        validation_result["cross_reference_matches"] = cross_ref_score
-        
-        # Calculate overall consistency score
-        if len(entities) > 0:
-            validation_result["consistency_score"] = (
-                validation_result["valid_entities"] / len(entities)
-            ) * 0.7 + (cross_ref_score / 100) * 0.3
-        
-        return validation_result
-    
-    async def _validate_entity_type(
-        self, 
-        entity_type: str, 
-        entities: List[ExtractedEntity]
-    ) -> Dict[str, Any]:
-        """Validate entities of a specific type."""
-        
-        validation = {"valid_count": 0, "invalid_count": 0, "errors": []}
-        
-        for entity in entities:
-            is_valid = True
-            
-            # Basic validation rules by type
-            if entity_type == "email":
-                if not re.match(r'^[^@]+@[^@]+\.[^@]+$', str(entity.value)):
-                    is_valid = False
-                    validation["errors"].append(f"Invalid email format: {entity.value}")
-            
-            elif entity_type == "phone":
-                # Remove non-digits and check length
-                digits = re.sub(r'\D', '', str(entity.value))
-                if len(digits) not in [10, 11]:
-                    is_valid = False
-                    validation["errors"].append(f"Invalid phone format: {entity.value}")
-            
-            elif entity_type == "url":
-                try:
-                    parsed = urlparse(str(entity.value))
-                    if not parsed.scheme or not parsed.netloc:
-                        is_valid = False
-                        validation["errors"].append(f"Invalid URL format: {entity.value}")
-                except:
-                    is_valid = False
-                    validation["errors"].append(f"Invalid URL format: {entity.value}")
-            
-            # Check confidence threshold
-            if entity.confidence < 0.5:
-                is_valid = False
-                validation["errors"].append(f"Low confidence entity: {entity.value} ({entity.confidence})")
-            
-            if is_valid:
-                validation["valid_count"] += 1
-            else:
-                validation["invalid_count"] += 1
-        
-        return validation
-    
-    async def _cross_reference_validation(
-        self, 
-        entities_by_type: Dict[str, List[ExtractedEntity]]
-    ) -> int:
-        """Validate entities through cross-referencing."""
-        
-        matches = 0
-        
-        # Compare entities across different URLs for consistency
-        for entity_type, entities in entities_by_type.items():
-            if len(entities) < 2:
-                continue
-            
-            # Group by URL
-            entities_by_url = {}
-            for entity in entities:
-                if entity.source_url not in entities_by_url:
-                    entities_by_url[entity.source_url] = []
-                entities_by_url[entity.source_url].append(entity)
-            
-            # Compare entities across URLs
-            urls = list(entities_by_url.keys())
-            for i in range(len(urls)):
-                for j in range(i + 1, len(urls)):
-                    url1_entities = entities_by_url[urls[i]]
-                    url2_entities = entities_by_url[urls[j]]
-                    
-                    # Find matching values
-                    for e1 in url1_entities:
-                        for e2 in url2_entities:
-                            if self._entities_match(e1, e2):
-                                matches += 1
-        
-        return matches
-    
-    def _entities_match(self, e1: ExtractedEntity, e2: ExtractedEntity) -> bool:
-        """Check if two entities match."""
-        if e1.entity_type != e2.entity_type:
-            return False
-        
-        # Exact match
-        if str(e1.value).lower().strip() == str(e2.value).lower().strip():
-            return True
-        
-        # Fuzzy match for text entities
-        if isinstance(e1.value, str) and isinstance(e2.value, str):
-            tokens1 = set(clean_tokens(e1.value.lower().split()))
-            tokens2 = set(clean_tokens(e2.value.lower().split()))
-            
-            if tokens1 and tokens2:
-                overlap = len(tokens1.intersection(tokens2))
-                union = len(tokens1.union(tokens2))
-                similarity = overlap / union if union > 0 else 0
-                return similarity > 0.8
-        
-        return False
-
-
-class MultiEntityExtractionService:
-    """
-    Advanced multi-entity extraction service.
-    
-    Provides sophisticated extraction capabilities including:
-    - Cross-URL entity discovery and linking
-    - Multiple extraction strategies
-    - Entity validation and consistency checking
-    - Relationship mapping between entities
-    - Temporal and semantic analysis
-    """
-    
-    def __init__(self):
-        """Initialize multi-entity extraction service."""
-        self.entity_linker = EntityLinker()
-        self.entity_extractor = EntityExtractor()
-        self.entity_validator = EntityValidator()
-        self.extraction_stats = {"total_requests": 0, "successful_extractions": 0}
-    
-    async def extract_multi_entity(
-        self, 
-        request: MultiEntityExtractionRequest
-    ) -> MultiEntityExtractionResult:
-        """
-        Perform multi-entity extraction across multiple URLs.
-        
-        Args:
-            request: Multi-entity extraction request
-            
-        Returns:
-            Comprehensive extraction results with entities and relationships
-        """
-        
-        start_time = time.time()
-        request_id = f"multi-extract-{int(time.time())}"
-        self.extraction_stats["total_requests"] += 1
-        
-        logger.info("multi_entity_extraction_started",
-                   request_id=request_id,
-                   urls=len(request.urls),
-                   strategy=request.extraction_strategy.value)
-        
-        try:
-            # Step 1: Discover related URLs
-            all_urls = request.urls.copy()
-            
-            if request.follow_links and len(request.urls) > 0:
-                logger.info("discovering_related_urls", request_id=request_id)
-                related_urls = await self.entity_linker.discover_related_urls(
-                    request.urls,
-                    request.extraction_strategy,
-                    request.max_related_urls
-                )
-                
-                # Add new URLs (avoid duplicates)
-                for url in related_urls:
-                    if url not in all_urls:
-                        all_urls.append(url)
-                
-                logger.info("related_urls_discovered", 
-                           request_id=request_id, 
-                           total_urls=len(all_urls))
-            
-            # Step 2: Scrape all URLs
-            logger.info("scraping_urls", request_id=request_id, urls=len(all_urls))
-            scraped_contents = await self._scrape_urls(all_urls)
-            
-            successful_scrapes = [c for c in scraped_contents if c.extraction_success]
-            logger.info("scraping_completed",
-                       request_id=request_id,
-                       successful=len(successful_scrapes),
-                       failed=len(scraped_contents) - len(successful_scrapes))
-            
-            # Step 3: Extract entities from each URL
-            logger.info("extracting_entities", request_id=request_id)
-            all_entities = []
-            
-            for content in successful_scrapes:
-                entities = await self.entity_extractor.extract_entities(
-                    content, 
-                    request.schema
-                )
-                all_entities.extend(entities)
-            
-            logger.info("entity_extraction_completed",
-                       request_id=request_id,
-                       entities_extracted=len(all_entities))
-            
-            # Step 4: Find relationships between entities
-            logger.info("analyzing_entity_relationships", request_id=request_id)
-            relationships = await self._analyze_relationships(all_entities, request)
-            
-            # Step 5: Validate entities if requested
-            validation_results = {}
-            if request.cross_validate:
-                logger.info("validating_entities", request_id=request_id)
-                validation_results = await self.entity_validator.validate_entities(
-                    all_entities
-                )
-            
-            # Step 6: Prepare extraction metadata
-            extraction_metadata = {
-                "urls_processed": len(all_urls),
-                "urls_successful": len(successful_scrapes),
-                "extraction_strategy": request.extraction_strategy.value,
-                "schema_fields": list(request.schema.get("properties", {}).keys()),
-                "processing_time": time.time() - start_time
-            }
-            
-            processing_time = time.time() - start_time
-            self.extraction_stats["successful_extractions"] += 1
-            
-            result = MultiEntityExtractionResult(
-                request_id=request_id,
-                entities=all_entities,
-                relations=relationships,
-                discovered_urls=all_urls,
-                validation_results=validation_results,
-                extraction_metadata=extraction_metadata,
-                processing_time=processing_time,
-                success=True
-            )
-            
-            logger.info("multi_entity_extraction_completed",
-                       request_id=request_id,
-                       entities=len(all_entities),
-                       relationships=len(relationships),
-                       processing_time=processing_time)
-            
-            return result
-            
-        except Exception as e:
-            processing_time = time.time() - start_time
-            error_msg = str(e)
-            
-            logger.error("multi_entity_extraction_failed",
-                        request_id=request_id,
-                        error=error_msg,
-                        processing_time=processing_time)
-            
-            return MultiEntityExtractionResult(
-                request_id=request_id,
-                entities=[],
-                relations=[],
-                discovered_urls=request.urls,
-                validation_results={},
-                extraction_metadata={},
-                processing_time=processing_time,
-                success=False,
-                errors=[error_msg]
-            )
-    
-    async def _scrape_urls(self, urls: List[str]) -> List[ScrapedContent]:
-        """Scrape multiple URLs efficiently."""
-        
-        scraping_service = await get_enhanced_scraping_service()
-        
-        # Configure scraping for entity extraction
-        config = ScrapingConfig(
-            urls=urls,
-            extract_text=True,
-            extract_links=True,
-            extract_metadata=True,
-            javascript_rendering=True,  # For dynamic content
-            wait_time=2
-        )
-        
-        # Scrape URLs in batches to manage resources
-        batch_size = 10
-        all_results = []
-        
-        for i in range(0, len(urls), batch_size):
-            batch_urls = urls[i:i + batch_size]
-            config.urls = batch_urls
-            
-            try:
-                batch_results = await scraping_service.scrape_urls_enhanced(
-                    batch_urls, config
-                )
-                all_results.extend(batch_results)
-                
-            except Exception as e:
-                logger.warning("batch_scraping_failed", 
-                             batch=i//batch_size + 1, 
-                             error=str(e))
-                
-                # Add failed results
-                for url in batch_urls:
-                    all_results.append(ScrapedContent(
-                        url=url,
-                        extraction_success=False,
-                        text="",
-                        metadata=ContentMetadata()
-                    ))
-        
-        return all_results
-    
-    async def _analyze_relationships(
-        self, 
-        entities: List[ExtractedEntity],
-        request: MultiEntityExtractionRequest
-    ) -> List[EntityRelation]:
-        """Analyze relationships between entities."""
-        
-        relationships = []
-        
-        # Group entities by URL
-        entities_by_url = {}
-        for entity in entities:
-            if entity.source_url not in entities_by_url:
-                entities_by_url[entity.source_url] = []
-            entities_by_url[entity.source_url].append(entity)
-        
-        # Find relationships within same URL (co-occurrence)
-        for url, url_entities in entities_by_url.items():
-            for i in range(len(url_entities)):
-                for j in range(i + 1, len(url_entities)):
-                    e1, e2 = url_entities[i], url_entities[j]
-                    
-                    relation = EntityRelation(
-                        source_url=url,
-                        target_url=url,
-                        relation_type="co_occurrence",
-                        confidence=0.6,
-                        evidence=[f"Found together on {url}"]
-                    )
-                    relationships.append(relation)
-        
-        # Find relationships across URLs
-        urls = list(entities_by_url.keys())
-        for i in range(len(urls)):
-            for j in range(i + 1, len(urls)):
-                url1_entities = entities_by_url[urls[i]]
-                url2_entities = entities_by_url[urls[j]]
-                
-                # Look for matching or similar entities
-                for e1 in url1_entities:
-                    for e2 in url2_entities:
-                        if self._entities_related(e1, e2):
-                            relation = EntityRelation(
-                                source_url=urls[i],
-                                target_url=urls[j],
-                                relation_type="cross_reference",
-                                confidence=0.8,
-                                evidence=[f"Similar entities: {e1.value} <-> {e2.value}"]
-                            )
-                            relationships.append(relation)
-        
-        return relationships
-    
-    def _entities_related(self, e1: ExtractedEntity, e2: ExtractedEntity) -> bool:
-        """Check if entities are related."""
-        # Same type and similar values
-        if e1.entity_type == e2.entity_type:
-            if isinstance(e1.value, str) and isinstance(e2.value, str):
-                # Simple similarity check
-                return e1.value.lower() in e2.value.lower() or e2.value.lower() in e1.value.lower()
-        
-        return False
-    
-    async def get_extraction_stats(self) -> Dict[str, Any]:
-        """Get extraction service statistics."""
-        return {
-            "extraction_stats": self.extraction_stats,
-            "cache_sizes": {
-                "entity_extractor": len(self.entity_extractor.extraction_cache),
-                "entity_linker": len(self.entity_linker.entity_cache)
-            },
-            "success_rate": (
-                self.extraction_stats["successful_extractions"] / 
-                max(1, self.extraction_stats["total_requests"])
-            )
-        }
-
-
-# Singleton instance
-_multi_entity_service: Optional[MultiEntityExtractionService] = None
-
-
-async def get_multi_entity_service() -> MultiEntityExtractionService:
-    """Get or create multi-entity extraction service instance."""
-    global _multi_entity_service
-    
-    if _multi_entity_service is None:
-        _multi_entity_service = MultiEntityExtractionService()
-    
-    return _multi_entity_service
diff --git a/apps/backend/app/services/multi_search.py b/apps/backend/app/services/multi_search.py
deleted file mode 100644
index 745f476..0000000
--- a/apps/backend/app/services/multi_search.py
+++ /dev/null
@@ -1,390 +0,0 @@
-"""
-Multi-provider search service inspired by Firecrawl's search architecture.
-
-Provides fallback capabilities across multiple search engines with smart provider selection.
-"""
-
-import asyncio
-import json
-from typing import Dict, List, Optional, Any, Union
-from enum import Enum
-from dataclasses import dataclass
-import httpx
-import structlog
-
-from app.config import get_settings
-from app.models.responses import SearchResult, SearchMetadata
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class SearchProvider(Enum):
-    """Available search providers in order of preference."""
-    FIRE_ENGINE = "fire_engine"
-    SERPER = "serper"
-    SEARCHAPI = "searchapi"  
-    SEARXNG = "searxng"
-    GOOGLE = "google"  # Fallback
-
-
-@dataclass
-class SearchOptions:
-    """Search configuration options."""
-    query: str
-    num_results: int = 10
-    lang: str = "en"
-    country: str = "us"
-    location: Optional[str] = None
-    tbs: Optional[str] = None  # Time-based search
-    filter: Optional[str] = None
-    advanced: bool = False
-    timeout: int = 30
-
-
-@dataclass
-class SearchProviderConfig:
-    """Configuration for each search provider."""
-    name: str
-    enabled: bool
-    api_key: Optional[str] = None
-    endpoint: Optional[str] = None
-    rate_limit: int = 100  # requests per minute
-    timeout: int = 30
-
-
-class MultiProviderSearchService:
-    """
-    Multi-provider search service with intelligent fallback.
-    
-    Provides unified search interface across multiple providers with:
-    - Automatic provider fallback
-    - Rate limiting and error handling  
-    - Result normalization and deduplication
-    - Performance monitoring
-    """
-    
-    def __init__(self):
-        """Initialize multi-provider search service."""
-        self.client = httpx.AsyncClient(timeout=30)
-        self.providers = self._configure_providers()
-        self.provider_stats = {provider: {"requests": 0, "errors": 0, "avg_latency": 0.0} 
-                              for provider in SearchProvider}
-        
-    def _configure_providers(self) -> Dict[SearchProvider, SearchProviderConfig]:
-        """Configure available search providers from environment."""
-        return {
-            SearchProvider.FIRE_ENGINE: SearchProviderConfig(
-                name="Fire Engine",
-                enabled=bool(settings.fire_engine_url),
-                endpoint=settings.fire_engine_url,
-                rate_limit=200
-            ),
-            SearchProvider.SERPER: SearchProviderConfig(
-                name="Serper",
-                enabled=bool(getattr(settings, 'serper_api_key', None)),
-                api_key=getattr(settings, 'serper_api_key', None),
-                rate_limit=150
-            ),
-            SearchProvider.SEARCHAPI: SearchProviderConfig(
-                name="SearchAPI",
-                enabled=bool(getattr(settings, 'searchapi_key', None)),
-                api_key=getattr(settings, 'searchapi_key', None),
-                rate_limit=100
-            ),
-            SearchProvider.SEARXNG: SearchProviderConfig(
-                name="SearXNG",
-                enabled=bool(settings.searxng_url),
-                endpoint=settings.searxng_url,
-                rate_limit=300
-            ),
-            SearchProvider.GOOGLE: SearchProviderConfig(
-                name="Google",
-                enabled=True,  # Always available as fallback
-                rate_limit=50
-            )
-        }
-    
-    async def search(self, options: SearchOptions) -> List[SearchResult]:
-        """
-        Perform multi-provider search with intelligent fallback.
-        
-        Args:
-            options: Search configuration
-            
-        Returns:
-            List of search results from the first successful provider
-        """
-        logger.info("multi_provider_search_started", 
-                   query=options.query, 
-                   providers=len([p for p in self.providers.values() if p.enabled]))
-        
-        # Try each provider in order until one succeeds
-        for provider_type, config in self.providers.items():
-            if not config.enabled:
-                continue
-                
-            try:
-                start_time = asyncio.get_event_loop().time()
-                results = await self._search_with_provider(provider_type, options)
-                
-                if results:
-                    # Update stats
-                    latency = asyncio.get_event_loop().time() - start_time
-                    self.provider_stats[provider_type]["requests"] += 1
-                    self.provider_stats[provider_type]["avg_latency"] = (
-                        (self.provider_stats[provider_type]["avg_latency"] + latency) / 2
-                    )
-                    
-                    logger.info("search_successful",
-                               provider=config.name,
-                               results=len(results),
-                               latency=latency)
-                    return results
-                
-            except Exception as e:
-                self.provider_stats[provider_type]["errors"] += 1
-                logger.warning("search_provider_failed",
-                             provider=config.name,
-                             error=str(e))
-                continue
-        
-        logger.error("all_search_providers_failed", query=options.query)
-        return []
-    
-    async def _search_with_provider(
-        self, 
-        provider: SearchProvider, 
-        options: SearchOptions
-    ) -> List[SearchResult]:
-        """Execute search with specific provider."""
-        
-        if provider == SearchProvider.FIRE_ENGINE:
-            return await self._fire_engine_search(options)
-        elif provider == SearchProvider.SERPER:
-            return await self._serper_search(options)
-        elif provider == SearchProvider.SEARCHAPI:
-            return await self._searchapi_search(options)
-        elif provider == SearchProvider.SEARXNG:
-            return await self._searxng_search(options)
-        else:  # Google fallback
-            return await self._google_search(options)
-    
-    async def _fire_engine_search(self, options: SearchOptions) -> List[SearchResult]:
-        """Search using Fire Engine API."""
-        config = self.providers[SearchProvider.FIRE_ENGINE]
-        
-        payload = {
-            "q": options.query,
-            "numResults": options.num_results,
-            "lang": options.lang,
-            "country": options.country,
-        }
-        
-        if options.location:
-            payload["location"] = options.location
-        if options.tbs:
-            payload["tbs"] = options.tbs
-        if options.filter:
-            payload["filter"] = options.filter
-            
-        response = await self.client.post(
-            f"{config.endpoint}/search",
-            json=payload,
-            headers={"Content-Type": "application/json"}
-        )
-        response.raise_for_status()
-        
-        data = response.json()
-        return self._normalize_fire_engine_results(data)
-    
-    async def _serper_search(self, options: SearchOptions) -> List[SearchResult]:
-        """Search using Serper API."""
-        config = self.providers[SearchProvider.SERPER]
-        
-        payload = {
-            "q": options.query,
-            "num": options.num_results,
-            "hl": options.lang,
-            "gl": options.country,
-        }
-        
-        if options.location:
-            payload["location"] = options.location
-        if options.tbs:
-            payload["tbs"] = options.tbs
-            
-        response = await self.client.post(
-            "https://google.serper.dev/search",
-            json=payload,
-            headers={
-                "X-API-KEY": config.api_key,
-                "Content-Type": "application/json"
-            }
-        )
-        response.raise_for_status()
-        
-        data = response.json()
-        return self._normalize_serper_results(data)
-    
-    async def _searchapi_search(self, options: SearchOptions) -> List[SearchResult]:
-        """Search using SearchAPI."""
-        config = self.providers[SearchProvider.SEARCHAPI]
-        
-        params = {
-            "engine": "google",
-            "q": options.query,
-            "num": options.num_results,
-            "hl": options.lang,
-            "gl": options.country,
-            "api_key": config.api_key
-        }
-        
-        if options.location:
-            params["location"] = options.location
-        if options.tbs:
-            params["tbs"] = options.tbs
-            
-        response = await self.client.get(
-            "https://www.searchapi.io/api/v1/search",
-            params=params
-        )
-        response.raise_for_status()
-        
-        data = response.json()
-        return self._normalize_searchapi_results(data)
-    
-    async def _searxng_search(self, options: SearchOptions) -> List[SearchResult]:
-        """Search using SearXNG instance."""
-        config = self.providers[SearchProvider.SEARXNG]
-        
-        params = {
-            "q": options.query,
-            "format": "json",
-            "categories": "general",
-            "language": options.lang,
-            "pageno": 1,
-        }
-        
-        response = await self.client.get(
-            f"{config.endpoint}/search",
-            params=params
-        )
-        response.raise_for_status()
-        
-        data = response.json()
-        return self._normalize_searxng_results(data, options.num_results)
-    
-    async def _google_search(self, options: SearchOptions) -> List[SearchResult]:
-        """Fallback Google search (simplified implementation)."""
-        # This would implement direct Google scraping as fallback
-        # For now, return empty results as this needs careful implementation
-        logger.warning("google_fallback_not_implemented")
-        return []
-    
-    def _normalize_fire_engine_results(self, data: Dict[str, Any]) -> List[SearchResult]:
-        """Normalize Fire Engine results."""
-        results = []
-        for item in data.get("results", []):
-            results.append(SearchResult(
-                title=item.get("title", ""),
-                url=item.get("url", ""),
-                description=item.get("description", ""),
-                engine="fire_engine",
-                score=item.get("score", 0.0)
-            ))
-        return results
-    
-    def _normalize_serper_results(self, data: Dict[str, Any]) -> List[SearchResult]:
-        """Normalize Serper results."""
-        results = []
-        for item in data.get("organic", []):
-            results.append(SearchResult(
-                title=item.get("title", ""),
-                url=item.get("link", ""),
-                description=item.get("snippet", ""),
-                engine="serper",
-                score=item.get("position", 0)
-            ))
-        return results
-    
-    def _normalize_searchapi_results(self, data: Dict[str, Any]) -> List[SearchResult]:
-        """Normalize SearchAPI results."""
-        results = []
-        for item in data.get("organic_results", []):
-            results.append(SearchResult(
-                title=item.get("title", ""),
-                url=item.get("link", ""),
-                description=item.get("snippet", ""),
-                engine="searchapi",
-                score=item.get("position", 0)
-            ))
-        return results
-    
-    def _normalize_searxng_results(self, data: Dict[str, Any], limit: int) -> List[SearchResult]:
-        """Normalize SearXNG results."""
-        results = []
-        for item in data.get("results", [])[:limit]:
-            results.append(SearchResult(
-                title=item.get("title", ""),
-                url=item.get("url", ""),
-                description=item.get("content", ""),
-                engine="searxng",
-                score=0.0
-            ))
-        return results
-    
-    async def get_provider_stats(self) -> Dict[str, Any]:
-        """Get performance statistics for all providers."""
-        return {
-            "providers": {
-                provider.value: {
-                    "config": {
-                        "name": config.name,
-                        "enabled": config.enabled,
-                        "rate_limit": config.rate_limit
-                    },
-                    "stats": self.provider_stats[provider]
-                }
-                for provider, config in self.providers.items()
-            }
-        }
-    
-    async def cleanup(self):
-        """Cleanup resources."""
-        if self.client:
-            await self.client.aclose()
-
-
-# Singleton instance
-_multi_search_service: Optional[MultiProviderSearchService] = None
-
-
-async def get_multi_search_service() -> MultiProviderSearchService:
-    """Get or create multi-provider search service instance."""
-    global _multi_search_service
-    
-    if _multi_search_service is None:
-        _multi_search_service = MultiProviderSearchService()
-    
-    return _multi_search_service
-
-
-# Convenience functions
-async def search_multi_provider(
-    query: str,
-    num_results: int = 10,
-    lang: str = "en",
-    country: str = "us",
-    **kwargs
-) -> List[SearchResult]:
-    """Convenience function for multi-provider search."""
-    service = await get_multi_search_service()
-    options = SearchOptions(
-        query=query,
-        num_results=num_results,
-        lang=lang,
-        country=country,
-        **kwargs
-    )
-    return await service.search(options)
diff --git a/apps/backend/app/services/pdf_processing.py b/apps/backend/app/services/pdf_processing.py
deleted file mode 100644
index 3e6d16a..0000000
--- a/apps/backend/app/services/pdf_processing.py
+++ /dev/null
@@ -1,617 +0,0 @@
-"""
-Advanced PDF processing system for document extraction and analysis.
-
-This module provides comprehensive PDF processing capabilities:
-- Multi-strategy PDF processing (Naive, Advanced)
-- PDF metadata extraction
-- Page-by-page content processing
-- Image extraction from PDFs
-- Text, HTML, and Markdown conversion
-- Layout analysis and structure preservation
-"""
-
-import io
-import re
-import base64
-import tempfile
-import time
-from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Any, Union
-from pathlib import Path
-from dataclasses import dataclass, field
-from datetime import datetime
-
-import structlog
-
-logger = structlog.get_logger(__name__)
-
-
-@dataclass
-class PDFMetadata:
-    """Metadata extracted from PDF document."""
-    title: Optional[str] = None
-    author: Optional[str] = None
-    subject: Optional[str] = None
-    producer: Optional[str] = None
-    creator: Optional[str] = None
-    created: Optional[datetime] = None
-    modified: Optional[datetime] = None
-    pages: int = 0
-    encrypted: bool = False
-    file_size: Optional[int] = None
-    version: Optional[str] = None
-
-
-@dataclass
-class PDFImage:
-    """Image extracted from PDF page."""
-    image_id: str
-    page_number: int
-    x: float = 0.0
-    y: float = 0.0
-    width: float = 0.0
-    height: float = 0.0
-    format: str = "PNG"
-    data: Optional[str] = None  # Base64 encoded
-    file_path: Optional[str] = None
-
-
-@dataclass
-class PDFPage:
-    """Single page from PDF document."""
-    page_number: int
-    raw_text: str = ""
-    markdown: str = ""
-    html: str = ""
-    images: List[PDFImage] = field(default_factory=list)
-    links: List[str] = field(default_factory=list)
-    layout: List[Dict[str, Any]] = field(default_factory=list)
-    width: float = 0.0
-    height: float = 0.0
-    rotation: int = 0
-
-
-@dataclass
-class PDFProcessResult:
-    """Complete result of PDF processing."""
-    metadata: PDFMetadata
-    pages: List[PDFPage]
-    processing_time: float = 0.0
-    success: bool = True
-    error: Optional[str] = None
-    version: str = "1.0"
-    
-    @property
-    def total_pages(self) -> int:
-        """Get total number of pages."""
-        return len(self.pages)
-    
-    @property
-    def total_text_length(self) -> int:
-        """Get total length of extracted text."""
-        return sum(len(page.raw_text) for page in self.pages)
-    
-    @property
-    def total_images(self) -> int:
-        """Get total number of images."""
-        return sum(len(page.images) for page in self.pages)
-
-
-class PDFProcessorStrategy(ABC):
-    """Abstract base class for PDF processing strategies."""
-    
-    @abstractmethod
-    def process(self, pdf_path: Path) -> PDFProcessResult:
-        """Process PDF file and return structured result."""
-        pass
-    
-    @abstractmethod
-    def process_from_bytes(self, pdf_bytes: bytes) -> PDFProcessResult:
-        """Process PDF from bytes and return structured result."""
-        pass
-
-
-class MockPDFProcessor(PDFProcessorStrategy):
-    """Mock PDF processor for environments without PDF libraries."""
-    
-    def __init__(self, **kwargs):
-        """Initialize mock processor."""
-        self.extract_images = kwargs.get('extract_images', True)
-        self.image_quality = kwargs.get('image_quality', 85)
-    
-    def process(self, pdf_path: Path) -> PDFProcessResult:
-        """Mock PDF processing from file path."""
-        start_time = time.time()
-        
-        try:
-            file_size = pdf_path.stat().st_size if pdf_path.exists() else 0
-        except Exception:
-            file_size = 0
-        
-        # Create mock result
-        metadata = PDFMetadata(
-            title="Mock PDF Document",
-            author="Unknown",
-            pages=3,  # Mock 3 pages
-            file_size=file_size,
-            version="1.4"
-        )
-        
-        pages = []
-        for i in range(1, 4):  # Mock 3 pages
-            page = PDFPage(
-                page_number=i,
-                raw_text=f"This is mock text from page {i} of the PDF document.\n\n"
-                         f"PDF processing requires additional dependencies that are not installed.\n"
-                         f"To enable full PDF processing, install: pip install PyPDF2 Pillow",
-                markdown=f"# Page {i}\n\nThis is mock text from page {i} of the PDF document.\n\n"
-                        f"PDF processing requires additional dependencies that are not installed.\n\n"
-                        f"To enable full PDF processing, install: `pip install PyPDF2 Pillow`",
-                width=612.0,  # Standard letter size
-                height=792.0
-            )
-            
-            # Mock image if image extraction is enabled
-            if self.extract_images and i == 1:  # Only on first page
-                mock_image = PDFImage(
-                    image_id=f"mock_image_{i}",
-                    page_number=i,
-                    width=100.0,
-                    height=100.0,
-                    format="PNG",
-                    data="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="  # 1x1 transparent PNG
-                )
-                page.images.append(mock_image)
-            
-            pages.append(page)
-        
-        processing_time = time.time() - start_time
-        
-        return PDFProcessResult(
-            metadata=metadata,
-            pages=pages,
-            processing_time=processing_time,
-            success=True,
-            version="mock-1.0"
-        )
-    
-    def process_from_bytes(self, pdf_bytes: bytes) -> PDFProcessResult:
-        """Mock PDF processing from bytes."""
-        # Create temporary file for mock processing
-        with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp_file:
-            tmp_file.write(pdf_bytes)
-            tmp_path = Path(tmp_file.name)
-        
-        try:
-            result = self.process(tmp_path)
-            result.metadata.file_size = len(pdf_bytes)
-            return result
-        finally:
-            # Clean up temp file
-            try:
-                tmp_path.unlink()
-            except Exception:
-                pass
-
-
-class NaivePDFProcessor(PDFProcessorStrategy):
-    """Naive PDF processor using PyPDF2 and basic text extraction."""
-    
-    def __init__(self, 
-                 image_dpi: int = 144,
-                 image_quality: int = 85,
-                 extract_images: bool = True,
-                 save_images_locally: bool = False,
-                 image_save_dir: Optional[Path] = None,
-                 batch_size: int = 4):
-        """Initialize PDF processor."""
-        self.image_dpi = image_dpi
-        self.image_quality = image_quality
-        self.extract_images = extract_images
-        self.save_images_locally = save_images_locally
-        self.image_save_dir = image_save_dir
-        self.batch_size = batch_size
-        self._temp_dir = None
-        
-        # Check for required dependencies
-        self._check_dependencies()
-    
-    def _check_dependencies(self):
-        """Check if required dependencies are available."""
-        try:
-            import PyPDF2  # noqa
-        except ImportError:
-            logger.warning(
-                "PyPDF2 not available. PDF processing will use mock implementation. "
-                "Install with: pip install PyPDF2"
-            )
-            self._use_mock = True
-            return
-        
-        if self.extract_images:
-            try:
-                from PIL import Image  # noqa
-            except ImportError:
-                logger.warning(
-                    "PIL/Pillow not available. Image extraction disabled. "
-                    "Install with: pip install Pillow"
-                )
-                self.extract_images = False
-        
-        self._use_mock = False
-    
-    def process(self, pdf_path: Path) -> PDFProcessResult:
-        """Process PDF file."""
-        if self._use_mock:
-            mock_processor = MockPDFProcessor(
-                extract_images=self.extract_images,
-                image_quality=self.image_quality
-            )
-            return mock_processor.process(pdf_path)
-        
-        return self._process_with_pypdf2(pdf_path)
-    
-    def process_from_bytes(self, pdf_bytes: bytes) -> PDFProcessResult:
-        """Process PDF from bytes."""
-        if self._use_mock:
-            mock_processor = MockPDFProcessor(
-                extract_images=self.extract_images,
-                image_quality=self.image_quality
-            )
-            return mock_processor.process_from_bytes(pdf_bytes)
-        
-        # Create temporary file
-        with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp_file:
-            tmp_file.write(pdf_bytes)
-            tmp_path = Path(tmp_file.name)
-        
-        try:
-            result = self._process_with_pypdf2(tmp_path)
-            result.metadata.file_size = len(pdf_bytes)
-            return result
-        finally:
-            # Clean up temp file
-            try:
-                tmp_path.unlink()
-            except Exception:
-                pass
-    
-    def _process_with_pypdf2(self, pdf_path: Path) -> PDFProcessResult:
-        """Process PDF using PyPDF2."""
-        from PyPDF2 import PdfReader
-        
-        start_time = time.time()
-        result = PDFProcessResult(
-            metadata=PDFMetadata(),
-            pages=[],
-            version="pypdf2-1.0"
-        )
-        
-        try:
-            with pdf_path.open('rb') as file:
-                reader = PdfReader(file)
-                result.metadata = self._extract_metadata(pdf_path, reader)
-                
-                # Setup image directory if needed
-                image_dir = None
-                if self.extract_images and self.save_images_locally:
-                    if self.image_save_dir:
-                        image_dir = Path(self.image_save_dir)
-                        image_dir.mkdir(exist_ok=True, parents=True)
-                    else:
-                        self._temp_dir = tempfile.mkdtemp(prefix='pdf_images_')
-                        image_dir = Path(self._temp_dir)
-                
-                # Process each page
-                for page_num, page in enumerate(reader.pages):
-                    try:
-                        pdf_page = self._process_page(page, page_num + 1, image_dir)
-                        result.pages.append(pdf_page)
-                    except Exception as e:
-                        logger.error(f"Error processing page {page_num + 1}: {str(e)}")
-                        # Create empty page on error
-                        error_page = PDFPage(
-                            page_number=page_num + 1,
-                            raw_text=f"Error processing page: {str(e)}"
-                        )
-                        result.pages.append(error_page)
-        
-        except Exception as e:
-            logger.error(f"Failed to process PDF {pdf_path}: {str(e)}")
-            result.success = False
-            result.error = str(e)
-        
-        finally:
-            # Clean up temp directory
-            if self._temp_dir and not self.image_save_dir:
-                import shutil
-                try:
-                    shutil.rmtree(self._temp_dir)
-                except Exception as e:
-                    logger.error(f"Failed to cleanup temp directory: {str(e)}")
-        
-        result.processing_time = time.time() - start_time
-        return result
-    
-    def _extract_metadata(self, pdf_path: Path, reader) -> PDFMetadata:
-        """Extract metadata from PDF."""
-        try:
-            file_stats = pdf_path.stat()
-            file_size = file_stats.st_size
-        except Exception:
-            file_size = None
-        
-        metadata = PDFMetadata(
-            pages=len(reader.pages),
-            encrypted=reader.is_encrypted,
-            file_size=file_size
-        )
-        
-        # Extract document info if available
-        if hasattr(reader, 'metadata') and reader.metadata:
-            doc_info = reader.metadata
-            
-            metadata.title = self._clean_metadata_string(doc_info.get('/Title'))
-            metadata.author = self._clean_metadata_string(doc_info.get('/Author'))
-            metadata.subject = self._clean_metadata_string(doc_info.get('/Subject'))
-            metadata.producer = self._clean_metadata_string(doc_info.get('/Producer'))
-            metadata.creator = self._clean_metadata_string(doc_info.get('/Creator'))
-            
-            # Parse dates
-            if '/CreationDate' in doc_info:
-                metadata.created = self._parse_pdf_date(doc_info['/CreationDate'])
-            
-            if '/ModDate' in doc_info:
-                metadata.modified = self._parse_pdf_date(doc_info['/ModDate'])
-        
-        return metadata
-    
-    def _clean_metadata_string(self, value) -> Optional[str]:
-        """Clean metadata string values."""
-        if not value:
-            return None
-        
-        # Handle PyPDF2 text objects
-        if hasattr(value, 'strip'):
-            cleaned = str(value).strip()
-            return cleaned if cleaned else None
-        
-        return str(value).strip() if value else None
-    
-    def _parse_pdf_date(self, date_str) -> Optional[datetime]:
-        """Parse PDF date string to datetime."""
-        if not date_str:
-            return None
-        
-        try:
-            # PDF date format: D:YYYYMMDDHHmmSSOHH'mm'
-            date_str = str(date_str)
-            if date_str.startswith('D:'):
-                date_str = date_str[2:]
-            
-            # Extract basic date components
-            if len(date_str) >= 14:
-                year = int(date_str[0:4])
-                month = int(date_str[4:6])
-                day = int(date_str[6:8])
-                hour = int(date_str[8:10])
-                minute = int(date_str[10:12])
-                second = int(date_str[12:14])
-                
-                return datetime(year, month, day, hour, minute, second)
-        
-        except Exception:
-            pass
-        
-        return None
-    
-    def _process_page(self, page, page_num: int, image_dir: Optional[Path]) -> PDFPage:
-        """Process a single PDF page."""
-        pdf_page = PDFPage(page_number=page_num)
-        
-        # Extract text
-        try:
-            raw_text = page.extract_text()
-            pdf_page.raw_text = self._clean_pdf_text(raw_text)
-            pdf_page.markdown = self._convert_text_to_markdown(pdf_page.raw_text)
-        except Exception as e:
-            logger.warning(f"Error extracting text from page {page_num}: {str(e)}")
-            pdf_page.raw_text = f"Error extracting text: {str(e)}"
-        
-        # Get page dimensions
-        try:
-            mediabox = page.mediabox
-            pdf_page.width = float(mediabox.width)
-            pdf_page.height = float(mediabox.height)
-            pdf_page.rotation = int(page.get('/Rotate', 0))
-        except Exception:
-            pass
-        
-        # Extract images if enabled
-        if self.extract_images:
-            try:
-                images = self._extract_images_from_page(page, page_num, image_dir)
-                pdf_page.images = images
-            except Exception as e:
-                logger.warning(f"Error extracting images from page {page_num}: {str(e)}")
-        
-        # Extract links
-        try:
-            links = self._extract_links_from_page(page)
-            pdf_page.links = links
-        except Exception as e:
-            logger.warning(f"Error extracting links from page {page_num}: {str(e)}")
-        
-        return pdf_page
-    
-    def _clean_pdf_text(self, text: str) -> str:
-        """Clean extracted PDF text."""
-        if not text:
-            return ""
-        
-        # Remove excessive whitespace
-        text = re.sub(r'\n\s*\n\s*\n', '\n\n', text)
-        text = re.sub(r' +', ' ', text)
-        
-        # Fix common PDF text extraction issues
-        text = text.replace('\x00', '')  # Remove null characters
-        text = text.replace('\ufffd', '')  # Remove replacement characters
-        
-        return text.strip()
-    
-    def _convert_text_to_markdown(self, text: str) -> str:
-        """Convert plain text to basic markdown."""
-        if not text:
-            return ""
-        
-        lines = text.split('\n')
-        markdown_lines = []
-        
-        for line in lines:
-            line = line.strip()
-            if not line:
-                markdown_lines.append('')
-                continue
-            
-            # Try to identify headers (all caps, short lines)
-            if len(line) < 100 and line.isupper() and len(line.split()) < 10:
-                markdown_lines.append(f'## {line.title()}')
-            else:
-                markdown_lines.append(line)
-        
-        return '\n'.join(markdown_lines)
-    
-    def _extract_images_from_page(self, page, page_num: int, image_dir: Optional[Path]) -> List[PDFImage]:
-        """Extract images from PDF page."""
-        images = []
-        
-        if not self.extract_images:
-            return images
-        
-        try:
-            # This is a simplified implementation
-            # In a full implementation, you would iterate through page objects
-            # and extract embedded images
-            
-            # For now, create a mock image to demonstrate the structure
-            mock_image = PDFImage(
-                image_id=f"img_{page_num}_1",
-                page_number=page_num,
-                width=200.0,
-                height=150.0,
-                format="PNG",
-                data="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
-            )
-            
-            # Save image file if directory provided
-            if image_dir:
-                image_path = image_dir / f"page_{page_num}_img_1.png"
-                mock_image.file_path = str(image_path)
-                
-                # In real implementation, save actual image data
-                try:
-                    with open(image_path, 'wb') as f:
-                        f.write(base64.b64decode(mock_image.data))
-                except Exception:
-                    pass
-            
-            images.append(mock_image)
-        
-        except Exception as e:
-            logger.warning(f"Error extracting images: {str(e)}")
-        
-        return images
-    
-    def _extract_links_from_page(self, page) -> List[str]:
-        """Extract links from PDF page."""
-        links = []
-        
-        try:
-            # Extract annotations that might be links
-            if '/Annots' in page:
-                annotations = page['/Annots']
-                for annotation in annotations:
-                    annotation_obj = annotation.get_object()
-                    if '/A' in annotation_obj and '/URI' in annotation_obj['/A']:
-                        uri = annotation_obj['/A']['/URI']
-                        if isinstance(uri, str):
-                            links.append(uri)
-        
-        except Exception as e:
-            logger.warning(f"Error extracting links: {str(e)}")
-        
-        return links
-
-
-# Factory function
-def create_pdf_processor(
-    processor_type: str = "naive",
-    config: Dict[str, Any] = None
-) -> PDFProcessorStrategy:
-    """Create PDF processor instance."""
-    config = config or {}
-    
-    processors = {
-        "naive": NaivePDFProcessor,
-        "mock": MockPDFProcessor
-    }
-    
-    if processor_type not in processors:
-        raise ValueError(f"Unknown processor type: {processor_type}. Available: {list(processors.keys())}")
-    
-    processor_class = processors[processor_type]
-    return processor_class(**config)
-
-
-# Convenience functions
-def process_pdf_file(
-    pdf_path: Union[str, Path],
-    processor_type: str = "naive",
-    config: Dict[str, Any] = None
-) -> PDFProcessResult:
-    """Process PDF file with specified processor."""
-    if isinstance(pdf_path, str):
-        pdf_path = Path(pdf_path)
-    
-    processor = create_pdf_processor(processor_type, config)
-    return processor.process(pdf_path)
-
-
-def process_pdf_bytes(
-    pdf_bytes: bytes,
-    processor_type: str = "naive",
-    config: Dict[str, Any] = None
-) -> PDFProcessResult:
-    """Process PDF bytes with specified processor."""
-    processor = create_pdf_processor(processor_type, config)
-    return processor.process_from_bytes(pdf_bytes)
-
-
-def extract_pdf_text(pdf_path: Union[str, Path]) -> str:
-    """Extract all text from PDF file."""
-    result = process_pdf_file(pdf_path)
-    if result.success:
-        return '\n\n'.join(page.raw_text for page in result.pages)
-    else:
-        return f"Error processing PDF: {result.error}"
-
-
-def pdf_to_markdown(pdf_path: Union[str, Path]) -> str:
-    """Convert PDF to markdown format."""
-    result = process_pdf_file(pdf_path)
-    if result.success:
-        markdown_parts = []
-        if result.metadata.title:
-            markdown_parts.append(f"# {result.metadata.title}")
-            markdown_parts.append("")
-        
-        for page in result.pages:
-            if page.markdown:
-                markdown_parts.append(f"## Page {page.page_number}")
-                markdown_parts.append("")
-                markdown_parts.append(page.markdown)
-                markdown_parts.append("")
-        
-        return '\n'.join(markdown_parts)
-    else:
-        return f"# Error Processing PDF\n\n{result.error}"
diff --git a/apps/backend/app/services/proxy_rotation.py b/apps/backend/app/services/proxy_rotation.py
deleted file mode 100644
index 0e4c34a..0000000
--- a/apps/backend/app/services/proxy_rotation.py
+++ /dev/null
@@ -1,534 +0,0 @@
-"""
-Advanced proxy rotation strategies for distributed web scraping.
-
-This module provides sophisticated proxy management:
-- Multiple rotation strategies (Round Robin, Random, Health-based)
-- Proxy health monitoring and failover
-- Automatic proxy validation and testing
-- Performance-based proxy selection
-- Geographic proxy distribution
-"""
-
-import asyncio
-import random
-import time
-from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Any, Set
-from dataclasses import dataclass, field
-from enum import Enum
-from urllib.parse import urlparse
-
-import httpx
-import structlog
-
-from app.services.browser_config import ProxyConfig
-
-logger = structlog.get_logger(__name__)
-
-
-class ProxyStatus(str, Enum):
-    """Proxy status enumeration."""
-    UNKNOWN = "unknown"
-    HEALTHY = "healthy"
-    SLOW = "slow"
-    UNHEALTHY = "unhealthy"
-    BLOCKED = "blocked"
-    DISABLED = "disabled"
-
-
-@dataclass
-class ProxyMetrics:
-    """Metrics for proxy performance tracking."""
-    proxy_id: str
-    total_requests: int = 0
-    successful_requests: int = 0
-    failed_requests: int = 0
-    avg_response_time: float = 0.0
-    last_used: Optional[float] = None
-    last_success: Optional[float] = None
-    last_failure: Optional[float] = None
-    consecutive_failures: int = 0
-    status: ProxyStatus = ProxyStatus.UNKNOWN
-    
-    @property
-    def success_rate(self) -> float:
-        """Calculate success rate."""
-        if self.total_requests == 0:
-            return 0.0
-        return (self.successful_requests / self.total_requests) * 100
-    
-    @property
-    def failure_rate(self) -> float:
-        """Calculate failure rate."""
-        return 100.0 - self.success_rate
-    
-    def update_success(self, response_time: float):
-        """Update metrics for successful request."""
-        self.total_requests += 1
-        self.successful_requests += 1
-        self.consecutive_failures = 0
-        self.last_used = time.time()
-        self.last_success = time.time()
-        
-        # Update average response time
-        if self.avg_response_time == 0:
-            self.avg_response_time = response_time
-        else:
-            # Exponential moving average
-            alpha = 0.1
-            self.avg_response_time = (alpha * response_time) + ((1 - alpha) * self.avg_response_time)
-        
-        # Update status based on response time
-        if response_time < 2.0:
-            self.status = ProxyStatus.HEALTHY
-        elif response_time < 5.0:
-            self.status = ProxyStatus.SLOW
-        else:
-            self.status = ProxyStatus.UNHEALTHY
-    
-    def update_failure(self, error_type: str = "unknown"):
-        """Update metrics for failed request."""
-        self.total_requests += 1
-        self.failed_requests += 1
-        self.consecutive_failures += 1
-        self.last_used = time.time()
-        self.last_failure = time.time()
-        
-        # Update status based on failure pattern
-        if self.consecutive_failures >= 5:
-            if "blocked" in error_type.lower() or "forbidden" in error_type.lower():
-                self.status = ProxyStatus.BLOCKED
-            else:
-                self.status = ProxyStatus.UNHEALTHY
-        elif self.failure_rate > 70:
-            self.status = ProxyStatus.UNHEALTHY
-
-
-@dataclass
-class ProxyInfo:
-    """Enhanced proxy information with metrics."""
-    config: ProxyConfig
-    metrics: ProxyMetrics = field(init=False)
-    region: Optional[str] = None
-    isp: Optional[str] = None
-    speed_tier: Optional[str] = None  # "fast", "medium", "slow"
-    
-    def __post_init__(self):
-        self.metrics = ProxyMetrics(proxy_id=self._generate_id())
-    
-    def _generate_id(self) -> str:
-        """Generate unique ID for proxy."""
-        return f"{self.config.ip}:{urlparse(self.config.server).port}"
-    
-    @property
-    def is_healthy(self) -> bool:
-        """Check if proxy is healthy enough to use."""
-        return self.metrics.status in [ProxyStatus.HEALTHY, ProxyStatus.SLOW, ProxyStatus.UNKNOWN]
-    
-    @property
-    def priority_score(self) -> float:
-        """Calculate priority score for proxy selection."""
-        base_score = 1.0
-        
-        # Success rate factor
-        success_factor = self.metrics.success_rate / 100.0
-        
-        # Response time factor (lower is better)
-        if self.metrics.avg_response_time > 0:
-            time_factor = max(0.1, 1.0 - (self.metrics.avg_response_time / 10.0))
-        else:
-            time_factor = 1.0
-        
-        # Consecutive failures penalty
-        failure_penalty = max(0.1, 1.0 - (self.metrics.consecutive_failures * 0.2))
-        
-        # Status factor
-        status_factors = {
-            ProxyStatus.HEALTHY: 1.0,
-            ProxyStatus.SLOW: 0.7,
-            ProxyStatus.UNKNOWN: 0.8,
-            ProxyStatus.UNHEALTHY: 0.2,
-            ProxyStatus.BLOCKED: 0.1,
-            ProxyStatus.DISABLED: 0.0
-        }
-        status_factor = status_factors.get(self.metrics.status, 0.5)
-        
-        return base_score * success_factor * time_factor * failure_penalty * status_factor
-
-
-class ProxyRotationStrategy(ABC):
-    """Abstract base class for proxy rotation strategies."""
-    
-    def __init__(self, proxies: List[ProxyConfig]):
-        """Initialize strategy with proxy list."""
-        self.proxy_infos: List[ProxyInfo] = [ProxyInfo(config=p) for p in proxies]
-        self.disabled_proxies: Set[str] = set()
-        
-        # Health checking
-        self.health_check_interval = 300  # 5 minutes
-        self.last_health_check = 0
-        self.health_check_url = "http://httpbin.org/ip"
-        self.health_check_timeout = 10.0
-    
-    @abstractmethod
-    async def get_next_proxy(self) -> Optional[ProxyConfig]:
-        """Get next proxy according to strategy."""
-        pass
-    
-    async def add_proxies(self, proxies: List[ProxyConfig]):
-        """Add new proxies to the rotation."""
-        new_proxy_infos = [ProxyInfo(config=p) for p in proxies]
-        self.proxy_infos.extend(new_proxy_infos)
-        logger.info(f"Added {len(proxies)} proxies to rotation")
-    
-    def remove_proxy(self, proxy_id: str):
-        """Remove proxy from rotation."""
-        self.proxy_infos = [p for p in self.proxy_infos if p.metrics.proxy_id != proxy_id]
-        self.disabled_proxies.discard(proxy_id)
-        logger.info(f"Removed proxy {proxy_id} from rotation")
-    
-    def disable_proxy(self, proxy_id: str):
-        """Temporarily disable a proxy."""
-        self.disabled_proxies.add(proxy_id)
-        for proxy_info in self.proxy_infos:
-            if proxy_info.metrics.proxy_id == proxy_id:
-                proxy_info.metrics.status = ProxyStatus.DISABLED
-        logger.warning(f"Disabled proxy {proxy_id}")
-    
-    def enable_proxy(self, proxy_id: str):
-        """Re-enable a disabled proxy."""
-        self.disabled_proxies.discard(proxy_id)
-        for proxy_info in self.proxy_infos:
-            if proxy_info.metrics.proxy_id == proxy_id:
-                if proxy_info.metrics.status == ProxyStatus.DISABLED:
-                    proxy_info.metrics.status = ProxyStatus.UNKNOWN
-        logger.info(f"Enabled proxy {proxy_id}")
-    
-    def record_success(self, proxy: ProxyConfig, response_time: float):
-        """Record successful proxy usage."""
-        proxy_id = f"{proxy.ip}:{urlparse(proxy.server).port}"
-        
-        for proxy_info in self.proxy_infos:
-            if proxy_info.metrics.proxy_id == proxy_id:
-                proxy_info.metrics.update_success(response_time)
-                break
-    
-    def record_failure(self, proxy: ProxyConfig, error_type: str = "unknown"):
-        """Record failed proxy usage."""
-        proxy_id = f"{proxy.ip}:{urlparse(proxy.server).port}"
-        
-        for proxy_info in self.proxy_infos:
-            if proxy_info.metrics.proxy_id == proxy_id:
-                proxy_info.metrics.update_failure(error_type)
-                
-                # Auto-disable if too many consecutive failures
-                if proxy_info.metrics.consecutive_failures >= 10:
-                    self.disable_proxy(proxy_id)
-                break
-    
-    async def health_check_proxies(self):
-        """Perform health check on all proxies."""
-        current_time = time.time()
-        
-        if current_time - self.last_health_check < self.health_check_interval:
-            return
-        
-        logger.info("Starting proxy health check")
-        
-        async def check_proxy(proxy_info: ProxyInfo) -> None:
-            try:
-                async with httpx.AsyncClient(
-                    proxies={"http://": proxy_info.config.server, "https://": proxy_info.config.server},
-                    timeout=self.health_check_timeout
-                ) as client:
-                    start_time = time.time()
-                    response = await client.get(self.health_check_url)
-                    response_time = time.time() - start_time
-                    
-                    if response.status_code == 200:
-                        proxy_info.metrics.update_success(response_time)
-                    else:
-                        proxy_info.metrics.update_failure(f"status_{response.status_code}")
-            
-            except Exception as e:
-                proxy_info.metrics.update_failure(str(e))
-        
-        # Check all proxies concurrently
-        tasks = [check_proxy(proxy_info) for proxy_info in self.proxy_infos]
-        await asyncio.gather(*tasks, return_exceptions=True)
-        
-        self.last_health_check = current_time
-        
-        # Log health check results
-        healthy_count = sum(1 for p in self.proxy_infos if p.is_healthy)
-        logger.info(f"Proxy health check completed: {healthy_count}/{len(self.proxy_infos)} proxies healthy")
-    
-    def get_available_proxies(self) -> List[ProxyInfo]:
-        """Get list of available (healthy and enabled) proxies."""
-        available = []
-        for proxy_info in self.proxy_infos:
-            if (proxy_info.metrics.proxy_id not in self.disabled_proxies and 
-                proxy_info.is_healthy):
-                available.append(proxy_info)
-        return available
-    
-    def get_proxy_stats(self) -> Dict[str, Any]:
-        """Get comprehensive proxy statistics."""
-        total_proxies = len(self.proxy_infos)
-        available_proxies = len(self.get_available_proxies())
-        disabled_proxies = len(self.disabled_proxies)
-        
-        status_counts = {}
-        for status in ProxyStatus:
-            status_counts[status.value] = sum(1 for p in self.proxy_infos if p.metrics.status == status)
-        
-        # Calculate aggregate metrics
-        total_requests = sum(p.metrics.total_requests for p in self.proxy_infos)
-        total_successes = sum(p.metrics.successful_requests for p in self.proxy_infos)
-        avg_success_rate = (total_successes / total_requests * 100) if total_requests > 0 else 0
-        
-        avg_response_time = sum(p.metrics.avg_response_time for p in self.proxy_infos if p.metrics.avg_response_time > 0)
-        if avg_response_time > 0:
-            active_proxies = sum(1 for p in self.proxy_infos if p.metrics.avg_response_time > 0)
-            avg_response_time = avg_response_time / active_proxies if active_proxies > 0 else 0
-        
-        return {
-            'total_proxies': total_proxies,
-            'available_proxies': available_proxies,
-            'disabled_proxies': disabled_proxies,
-            'status_distribution': status_counts,
-            'aggregate_metrics': {
-                'total_requests': total_requests,
-                'success_rate': avg_success_rate,
-                'avg_response_time': avg_response_time
-            },
-            'top_performers': [
-                {
-                    'proxy_id': p.metrics.proxy_id,
-                    'success_rate': p.metrics.success_rate,
-                    'avg_response_time': p.metrics.avg_response_time,
-                    'priority_score': p.priority_score
-                }
-                for p in sorted(self.proxy_infos, key=lambda x: x.priority_score, reverse=True)[:5]
-            ]
-        }
-
-
-class RoundRobinProxyStrategy(ProxyRotationStrategy):
-    """Round-robin proxy rotation strategy."""
-    
-    def __init__(self, proxies: List[ProxyConfig]):
-        """Initialize round-robin strategy."""
-        super().__init__(proxies)
-        self.current_index = 0
-    
-    async def get_next_proxy(self) -> Optional[ProxyConfig]:
-        """Get next proxy using round-robin selection."""
-        # Perform health check if needed
-        await self.health_check_proxies()
-        
-        available_proxies = self.get_available_proxies()
-        if not available_proxies:
-            logger.warning("No available proxies for round-robin selection")
-            return None
-        
-        # Select proxy using round-robin
-        proxy_info = available_proxies[self.current_index % len(available_proxies)]
-        self.current_index += 1
-        
-        return proxy_info.config
-
-
-class RandomProxyStrategy(ProxyRotationStrategy):
-    """Random proxy rotation strategy."""
-    
-    async def get_next_proxy(self) -> Optional[ProxyConfig]:
-        """Get next proxy using random selection."""
-        # Perform health check if needed
-        await self.health_check_proxies()
-        
-        available_proxies = self.get_available_proxies()
-        if not available_proxies:
-            logger.warning("No available proxies for random selection")
-            return None
-        
-        # Select random proxy
-        proxy_info = random.choice(available_proxies)
-        return proxy_info.config
-
-
-class WeightedProxyStrategy(ProxyRotationStrategy):
-    """Weighted proxy rotation based on performance metrics."""
-    
-    async def get_next_proxy(self) -> Optional[ProxyConfig]:
-        """Get next proxy using weighted selection based on performance."""
-        # Perform health check if needed
-        await self.health_check_proxies()
-        
-        available_proxies = self.get_available_proxies()
-        if not available_proxies:
-            logger.warning("No available proxies for weighted selection")
-            return None
-        
-        # Calculate weights based on priority scores
-        weights = [max(0.1, proxy_info.priority_score) for proxy_info in available_proxies]
-        total_weight = sum(weights)
-        
-        if total_weight == 0:
-            # Fallback to random selection
-            proxy_info = random.choice(available_proxies)
-        else:
-            # Weighted random selection
-            r = random.uniform(0, total_weight)
-            cumulative_weight = 0
-            proxy_info = available_proxies[0]  # Default fallback
-            
-            for i, weight in enumerate(weights):
-                cumulative_weight += weight
-                if r <= cumulative_weight:
-                    proxy_info = available_proxies[i]
-                    break
-        
-        return proxy_info.config
-
-
-class GeographicProxyStrategy(ProxyRotationStrategy):
-    """Geographic-based proxy rotation strategy."""
-    
-    def __init__(self, proxies: List[ProxyConfig], preferred_regions: List[str] = None):
-        """
-        Initialize geographic strategy.
-        
-        Args:
-            proxies: List of proxy configurations
-            preferred_regions: Preferred regions for proxy selection
-        """
-        super().__init__(proxies)
-        self.preferred_regions = preferred_regions or []
-        
-        # Set regions for proxies (in real implementation, would use IP geolocation)
-        self._assign_regions()
-    
-    def _assign_regions(self):
-        """Assign regions to proxies (mock implementation)."""
-        mock_regions = ["us-east", "us-west", "eu-west", "ap-southeast"]
-        
-        for proxy_info in self.proxy_infos:
-            # In real implementation, would use IP geolocation service
-            proxy_info.region = random.choice(mock_regions)
-    
-    async def get_next_proxy(self) -> Optional[ProxyConfig]:
-        """Get next proxy with geographic preference."""
-        # Perform health check if needed
-        await self.health_check_proxies()
-        
-        available_proxies = self.get_available_proxies()
-        if not available_proxies:
-            logger.warning("No available proxies for geographic selection")
-            return None
-        
-        # Filter by preferred regions if specified
-        if self.preferred_regions:
-            preferred_proxies = [
-                p for p in available_proxies 
-                if p.region in self.preferred_regions
-            ]
-            if preferred_proxies:
-                available_proxies = preferred_proxies
-        
-        # Use weighted selection within geographic constraints
-        weights = [max(0.1, proxy_info.priority_score) for proxy_info in available_proxies]
-        total_weight = sum(weights)
-        
-        if total_weight == 0:
-            proxy_info = random.choice(available_proxies)
-        else:
-            r = random.uniform(0, total_weight)
-            cumulative_weight = 0
-            proxy_info = available_proxies[0]
-            
-            for i, weight in enumerate(weights):
-                cumulative_weight += weight
-                if r <= cumulative_weight:
-                    proxy_info = available_proxies[i]
-                    break
-        
-        return proxy_info.config
-
-
-# Factory function
-def create_proxy_strategy(
-    strategy_type: str,
-    proxies: List[ProxyConfig],
-    config: Dict[str, Any] = None
-) -> ProxyRotationStrategy:
-    """
-    Create proxy rotation strategy.
-    
-    Args:
-        strategy_type: Type of strategy ("round_robin", "random", "weighted", "geographic")
-        proxies: List of proxy configurations
-        config: Additional strategy configuration
-        
-    Returns:
-        Configured proxy rotation strategy
-    """
-    config = config or {}
-    
-    strategies = {
-        "round_robin": RoundRobinProxyStrategy,
-        "random": RandomProxyStrategy,
-        "weighted": WeightedProxyStrategy,
-        "geographic": GeographicProxyStrategy
-    }
-    
-    if strategy_type not in strategies:
-        raise ValueError(f"Unknown strategy type: {strategy_type}. Available: {list(strategies.keys())}")
-    
-    strategy_class = strategies[strategy_type]
-    
-    # Handle special configuration for geographic strategy
-    if strategy_type == "geographic":
-        preferred_regions = config.get("preferred_regions", [])
-        return strategy_class(proxies, preferred_regions)
-    else:
-        return strategy_class(proxies)
-
-
-# Convenience functions
-def create_proxy_list_from_strings(proxy_strings: List[str]) -> List[ProxyConfig]:
-    """Create ProxyConfig list from string representations."""
-    proxies = []
-    for proxy_str in proxy_strings:
-        try:
-            proxy_config = ProxyConfig.from_string(proxy_str)
-            proxies.append(proxy_config)
-        except Exception as e:
-            logger.warning(f"Invalid proxy string '{proxy_str}': {str(e)}")
-    
-    return proxies
-
-
-async def test_proxy_rotation(
-    strategy: ProxyRotationStrategy,
-    num_requests: int = 10
-) -> Dict[str, Any]:
-    """Test proxy rotation strategy with multiple requests."""
-    results = {
-        'total_requests': num_requests,
-        'proxy_usage': {},
-        'errors': []
-    }
-    
-    for i in range(num_requests):
-        try:
-            proxy = await strategy.get_next_proxy()
-            if proxy:
-                proxy_id = f"{proxy.ip}:{urlparse(proxy.server).port}"
-                results['proxy_usage'][proxy_id] = results['proxy_usage'].get(proxy_id, 0) + 1
-            else:
-                results['errors'].append(f"Request {i}: No proxy available")
-        except Exception as e:
-            results['errors'].append(f"Request {i}: {str(e)}")
-    
-    return results
diff --git a/apps/backend/app/services/puppeteer_client.py b/apps/backend/app/services/puppeteer_client.py
deleted file mode 100644
index f4d45bd..0000000
--- a/apps/backend/app/services/puppeteer_client.py
+++ /dev/null
@@ -1,33 +0,0 @@
-"""
-Simple client for an external Puppeteer render service.
-
-Expected service API:
-POST /render { url, waitUntil, timeout, screenshot, pdf, userAgent, headers?, cookies?, proxy? }
-Response JSON: { html: string, finalUrl?: string, screenshot?: string(base64), pdf?: string(base64) }
-"""
-from typing import Any, Dict, Optional
-import httpx
-from app.config import get_settings
-import structlog
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class PuppeteerClient:
-    def __init__(self, base_url: Optional[str] = None, timeout_seconds: Optional[int] = None):
-        self.base_url = (base_url or str(settings.puppeteer_service_url)).rstrip("/")
-        self.timeout = timeout_seconds or settings.puppeteer_timeout
-
-    async def render(self, payload: Dict[str, Any]) -> Dict[str, Any]:
-        url = f"{self.base_url}/render"
-        async with httpx.AsyncClient(timeout=httpx.Timeout(self.timeout)) as client:
-            resp = await client.post(url, json=payload)
-            resp.raise_for_status()
-            data = resp.json()
-            # Basic validation
-            if not isinstance(data, dict) or "html" not in data:
-                raise ValueError("Invalid response from Puppeteer service: missing 'html'")
-            return data
-
-
diff --git a/apps/backend/app/services/scraping.py b/apps/backend/app/services/scraping.py
deleted file mode 100644
index e1bd246..0000000
--- a/apps/backend/app/services/scraping.py
+++ /dev/null
@@ -1,815 +0,0 @@
-"""
-Content scraping service using BeautifulSoup4.
-"""
-import asyncio
-import re
-from typing import Dict, List, Optional, Tuple, Any
-from urllib.parse import urljoin, urlparse
-from urllib.robotparser import RobotFileParser
-import httpx
-from httpx import AsyncClient, HTTPError, TimeoutException
-from bs4 import BeautifulSoup, Comment
-import chardet
-# from readability import Readability  # Optional: install readability-lxml for better extraction
-import json
-from datetime import datetime
-import hashlib
-
-from app.config import get_settings
-from app.models.responses import ScrapedContent, ContentMetadata
-from app.models.requests import ScrapingConfig
-from app.utils.text_processing import sanitize_text
-from app.utils.text_processing import (
-    sanitize_text, 
-    detect_language, 
-    calculate_text_quality,
-    extract_keywords
-)
-import structlog
-from app.services.cache import get_cache_service
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class ContentScrapingService:
-    """Service for web content extraction using BeautifulSoup4."""
-    
-    def __init__(self):
-        self.user_agent = settings.scraping_user_agent
-        self.timeout = settings.scraping_timeout
-        self.max_concurrent = settings.scraping_max_concurrent
-        self.respect_robots = settings.scraping_respect_robots_txt
-        self.min_delay = settings.scraping_min_delay_seconds
-        self._client: Optional[AsyncClient] = None
-        self._robots_cache: Dict[str, RobotFileParser] = {}
-        self._semaphore = asyncio.Semaphore(self.max_concurrent)
-        
-    async def __aenter__(self):
-        """Async context manager entry."""
-        await self.initialize()
-        return self
-        
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        await self.close()
-        
-    async def initialize(self):
-        """Initialize HTTP client with connection pooling."""
-        if not self._client:
-            self._client = AsyncClient(
-                timeout=httpx.Timeout(self.timeout),
-                limits=httpx.Limits(
-                    max_keepalive_connections=20,
-                    max_connections=50,
-                ),
-                headers={
-                    "User-Agent": self.user_agent,
-                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                    "Accept-Language": "en-US,en;q=0.9",
-                    "Accept-Encoding": "gzip, deflate",
-                    "DNT": "1",
-                    "Connection": "keep-alive",
-                    "Upgrade-Insecure-Requests": "1"
-                },
-                follow_redirects=True,
-                max_redirects=5
-            )
-            
-    async def close(self):
-        """Close HTTP client connections."""
-        if self._client:
-            await self._client.aclose()
-            self._client = None
-            
-    async def scrape_urls(
-        self, 
-        urls: List[str], 
-        config: Optional[ScrapingConfig] = None
-    ) -> List[ScrapedContent]:
-        """
-        Scrape multiple URLs concurrently.
-        
-        Args:
-            urls: List of URLs to scrape
-            config: Scraping configuration
-            
-        Returns:
-            List of ScrapedContent objects
-        """
-        if not self._client:
-            await self.initialize()
-            
-        # Per-host concurrency and rate limiting
-        host_semaphores: Dict[str, asyncio.Semaphore] = {}
-        host_last_hit: Dict[str, float] = {}
-        per_host_concurrency = getattr(config, 'per_host_concurrency', 2) if config else 2
-        hits_per_sec = getattr(config, 'hits_per_sec', 0.0) if config else 0.0
-
-        loop = asyncio.get_event_loop()
-
-        def get_host(url: str) -> str:
-            try:
-                return urlparse(url).netloc.lower()
-            except Exception:
-                return ""
-
-        async def run_with_limits(url: str) -> ScrapedContent:
-            host = get_host(url)
-            if host and host not in host_semaphores:
-                host_semaphores[host] = asyncio.Semaphore(max(1, per_host_concurrency))
-
-            # Rate limiting per host
-            if hits_per_sec and hits_per_sec > 0:
-                min_interval = 1.0 / hits_per_sec
-                last = host_last_hit.get(host, 0.0)
-                now = loop.time()
-                wait_for = (last + min_interval) - now
-                if wait_for and wait_for > 0:
-                    await asyncio.sleep(wait_for)
-
-            if host:
-                async with host_semaphores[host]:
-                    host_last_hit[host] = loop.time()
-                    return await self._scrape_single_url(url, config)
-            else:
-                host_last_hit[host] = loop.time()
-                return await self._scrape_single_url(url, config)
-
-        # Create tasks for concurrent scraping with host-aware throttling
-        tasks = [run_with_limits(url) for url in urls]
-            
-        # Wait for all tasks with proper error handling
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-        
-        # Process results
-        scraped_contents = []
-        for idx, result in enumerate(results):
-            if isinstance(result, Exception):
-                logger.error(
-                    "scraping_task_failed",
-                    url=urls[idx],
-                    error=str(result),
-                    error_type=type(result).__name__
-                )
-                # Create error result
-                scraped_contents.append(
-                    ScrapedContent(
-                        url=urls[idx],
-                        title=None,
-                        text="",
-                        extraction_success=False,
-                        extraction_time_ms=0,
-                        word_count=0,
-                        metadata=ContentMetadata(),
-                        error_message=str(result),
-                        content_quality_score=0.0
-                    )
-                )
-            else:
-                scraped_contents.append(result)
-                
-        return scraped_contents
-        
-    async def _scrape_single_url(
-        self, 
-        url: str, 
-        config: Optional[ScrapingConfig] = None
-    ) -> ScrapedContent:
-        """Scrape a single URL with rate limiting."""
-        async with self._semaphore:
-            # JS rendering path: attempt Puppeteer microservice fetch
-            if config and (config.javascript_rendering or config.js_mode) and settings.puppeteer_enabled:
-                try:
-                    page = await self._fetch_with_puppeteer(url, config)
-                    if page:
-                        return await self._scrape_page_payload(url, page["html"], config, page)
-                except Exception as e:
-                    logger.warning("puppeteer_fetch_failed", url=url, error=str(e))
-                    # Fallback to HTTPX+BS4
-            # Check robots.txt if enabled
-            if self.respect_robots and not await self._check_robots_txt(url):
-                logger.warning("scraping_blocked_by_robots", url=url)
-                return ScrapedContent(
-                    url=url,
-                    title=None,
-                    text="",
-                    extraction_success=False,
-                    extraction_time_ms=0,
-                    word_count=0,
-                    metadata=ContentMetadata(),
-                    error_message="Blocked by robots.txt",
-                    content_quality_score=0.0
-                )
-                
-            # Add delay between requests
-            await asyncio.sleep(self.min_delay)
-            
-            # Scrape the URL (HTTPX fetch)
-            return await self._scrape_url(url, config)
-            
-    async def _scrape_url(
-        self, 
-        url: str, 
-        config: Optional[ScrapingConfig] = None
-    ) -> ScrapedContent:
-        """Scrape and extract content from a single URL."""
-        start_time = asyncio.get_event_loop().time()
-        
-        try:
-            html_content: Optional[str] = None
-            cache_mode = getattr(config, 'cache_mode', 'enabled') if config else 'enabled'
-            # Cache read
-            if cache_mode in ("enabled", "read_only"):
-                try:
-                    cache = await get_cache_service()
-                    cached_html = await cache.get_cached_url_content(url)
-                    if cached_html:
-                        html_content = cached_html
-                        logger.info("url_cache_hit", url=url)
-                except Exception as e:
-                    logger.warning("url_cache_read_failed", url=url, error=str(e))
-            # Build request headers
-            headers = dict(self._client.headers)
-            if config and config.headers:
-                headers.update(config.headers)
-                
-            # Make request
-            if not html_content:
-                response = await self._client.get(
-                    url,
-                    headers=headers,
-                    cookies=config.cookies if config else None
-                )
-                response.raise_for_status()
-                
-                # Detect encoding
-                encoding = self._detect_encoding(response)
-                html_content = response.content.decode(encoding, errors='ignore')
-                # Cache write
-                if cache_mode in ("enabled", "write_only") and config and getattr(config, 'cache_ttl', 0) > 0:
-                    try:
-                        cache = await get_cache_service()
-                        await cache.set_cached_url_content(url, html_content, ttl=config.cache_ttl)
-                        logger.info("url_cache_write", url=url)
-                    except Exception as e:
-                        logger.warning("url_cache_write_failed", url=url, error=str(e))
-            
-            # Parse HTML
-            soup = BeautifulSoup(html_content, 'lxml')
-            
-            # Remove script and style elements
-            for element in soup(['script', 'style', 'noscript']):
-                element.decompose()
-                
-            # Extract content based on configuration
-            if config and config.selectors:
-                extracted = await self._extract_with_selectors(soup, config.selectors)
-            else:
-                extracted = await self._extract_main_content(soup, url)
-                
-            # Extract metadata
-            metadata = await self.extract_metadata(soup, url)
-            
-            # Extract images if requested
-            images = []
-            if not config or config.extract_images:
-                images = self._extract_images(soup, url)
-                
-            # Extract links if requested
-            links = []
-            if not config or config.extract_links:
-                links = self._extract_links(soup, url)
-                # Optional link head enrichment and scoring
-                if config and getattr(config, 'link_head', False) and links:
-                    try:
-                        links = await self._enrich_and_score_links(links, config)
-                    except Exception as e:
-                        logger.warning("link_enrichment_failed", url=url, error=str(e))
-                
-            # Detect language
-            language = detect_language(extracted['text'])
-            
-            # Calculate quality score
-            quality_score = calculate_text_quality(extracted['text'])
-            
-            # Calculate processing time
-            extraction_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000)
-            
-            # Optionally return Markdown format when requested via config.response_format
-            if config and getattr(config, 'response_format', 'json') == 'markdown':
-                markdown_text = self._to_markdown(html_content, base_url=str(url))
-                # Overwrite text with markdown output for markdown mode
-                text_out = markdown_text
-            else:
-                text_out = extracted['text']
-
-            return ScrapedContent(
-                url=url,
-                title=extracted.get('title', metadata.title),
-                text=text_out,
-                html=html_content if config and hasattr(config, 'include_html') and config.include_html else None,
-                images=images,
-                links=links,
-                metadata=metadata,
-                extraction_success=True,
-                extraction_time_ms=extraction_time_ms,
-                word_count=len(text_out.split()) if text_out else 0,
-                language_detected=language,
-                content_quality_score=quality_score
-            )
-            
-        except Exception as e:
-            extraction_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000)
-            logger.error("scraping_error", url=url, error=str(e))
-            
-            return ScrapedContent(
-                url=url,
-                title=None,
-                text="",
-                extraction_success=False,
-                extraction_time_ms=extraction_time_ms,
-                word_count=0,
-                metadata=ContentMetadata(),
-                error_message=str(e),
-                content_quality_score=0.0
-            )
-
-    async def _fetch_with_puppeteer(self, url: str, config: ScrapingConfig) -> Optional[Dict[str, Any]]:
-        """Fetch page via Puppeteer microservice. Returns dict with html, screenshot, pdf, final_url."""
-        client_timeout = config.wait_time + settings.puppeteer_timeout
-        params = {
-            "url": url,
-            "waitUntil": config.wait_until or settings.puppeteer_default_wait_until,
-            "timeout": settings.puppeteer_timeout * 1000,
-            "screenshot": bool(getattr(config, "screenshot", False)),
-            "pdf": bool(getattr(config, "pdf", False)),
-            "userAgent": config.user_agent or self.user_agent,
-        }
-        if config.headers:
-            params["headers"] = config.headers
-        if config.cookies:
-            params["cookies"] = config.cookies
-        if getattr(config, "proxy", None):
-            params["proxy"] = config.proxy
-
-        service_url = str(settings.puppeteer_service_url).rstrip('/') + "/render"
-
-        async with AsyncClient(timeout=httpx.Timeout(client_timeout)) as client:
-            resp = await client.post(service_url, json=params)
-            resp.raise_for_status()
-            data = resp.json()
-            return data
-
-    async def _scrape_page_payload(self, url: str, html: str, config: ScrapingConfig, page_meta: Optional[Dict[str, Any]] = None) -> ScrapedContent:
-        """Scrape using provided HTML payload (from Puppeteer)."""
-        start_time = asyncio.get_event_loop().time()
-        soup = BeautifulSoup(html or "", 'lxml')
-        for element in soup(['script', 'style', 'noscript']):
-            element.decompose()
-
-        if config and config.selectors:
-            extracted = await self._extract_with_selectors(soup, config.selectors)
-        else:
-            extracted = await self._extract_main_content(soup, url)
-
-        metadata = await self.extract_metadata(soup, url)
-
-        images = []
-        if not config or config.extract_images:
-            images = self._extract_images(soup, url)
-
-        links = []
-        if not config or config.extract_links:
-            links = self._extract_links(soup, url)
-            # Optional link head enrichment and scoring
-            if getattr(config, 'link_head', False) and links:
-                try:
-                    links = await self._enrich_and_score_links(links, config)
-                except Exception as e:
-                    logger.warning("link_enrichment_failed", url=url, error=str(e))
-
-        # Optionally return Markdown
-        if getattr(config, 'response_format', 'json') == 'markdown':
-            markdown_text = self._to_markdown(html, base_url=str(url))
-            text_out = markdown_text
-        else:
-            text_out = extracted['text']
-
-        language = detect_language(text_out)
-        quality_score = calculate_text_quality(text_out)
-        extraction_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000)
-
-        html_out = html if config and getattr(config, 'include_html', False) else None
-        return ScrapedContent(
-            url=url,
-            title=extracted.get('title', metadata.title),
-            text=text_out,
-            html=html_out,
-            images=images,
-            links=links,
-            metadata=metadata,
-            extraction_success=True,
-            extraction_time_ms=extraction_time_ms,
-            word_count=len(text_out.split()) if text_out else 0,
-            language_detected=language,
-            content_quality_score=quality_score
-        )
-
-    def _to_markdown(self, input_html: str, base_url: str = "") -> str:
-        """Convert HTML to Markdown (simple) with absolute links. Lightweight alternative to crawl4ai's generator."""
-        try:
-            from bs4 import BeautifulSoup
-            from urllib.parse import urljoin
-            soup = BeautifulSoup(input_html or "", 'lxml')
-            # Remove script/style
-            for element in soup(['script', 'style', 'noscript']):
-                element.decompose()
-            # Convert links to absolute and inline markdown-like refs
-            for a in soup.find_all('a', href=True):
-                href = a['href']
-                if base_url and not href.startswith(('http://', 'https://', 'mailto:')):
-                    a['href'] = urljoin(base_url, href)
-            text = soup.get_text('\n')
-            text = sanitize_text(text)
-            return text
-        except Exception as e:
-            logger.warning("markdown_conversion_failed", error=str(e))
-            return sanitize_text(BeautifulSoup(input_html or "", 'lxml').get_text('\n'))
-
-    async def _enrich_and_score_links(self, links: List[str], config: ScrapingConfig) -> List[str]:
-        """Fetch HEAD/title for links and apply simple relevance scoring/filtering."""
-        max_links = max(1, int(getattr(config, 'link_max', 100)))
-        concurrency = max(1, int(getattr(config, 'link_enrichment_concurrency', 8)))
-        timeout_s = max(1, int(getattr(config, 'link_timeout', 5)))
-        query = getattr(config, 'link_score_query', None)
-        threshold = getattr(config, 'link_score_threshold', None)
-
-        targets = links[:max_links]
-        sem = asyncio.Semaphore(concurrency)
-
-        async def fetch_title(u: str) -> Dict[str, Any]:
-            async with sem:
-                try:
-                    async with AsyncClient(timeout=httpx.Timeout(timeout_s)) as client:
-                        resp = await client.get(u, headers={"Accept": "text/html,application/xhtml+xml"})
-                        ok = resp.status_code < 400
-                        title_text = ""
-                        if ok and resp.headers.get("content-type", "").startswith("text/html"):
-                            try:
-                                s = BeautifulSoup(resp.text, 'lxml')
-                                t = s.find('title')
-                                if t:
-                                    title_text = sanitize_text(t.get_text())
-                            except Exception:
-                                title_text = ""
-                        return {"url": u, "ok": ok, "title": title_text}
-                except Exception:
-                    return {"url": u, "ok": False, "title": ""}
-
-        results = await asyncio.gather(*[fetch_title(u) for u in targets])
-
-        def score(item: Dict[str, Any]) -> float:
-            if not item.get("ok"):
-                return 0.0
-            if not query:
-                return 1.0
-            terms = [t.lower() for t in str(query).split() if len(t) > 2]
-            if not terms:
-                return 1.0
-            text = f"{item.get('title','')} {item.get('url','')}".lower()
-            hits = sum(1 for t in terms if t in text)
-            return min(1.0, hits / max(1, len(terms)))
-
-        scored = [(score(item), item["url"]) for item in results]
-        if threshold is not None:
-            try:
-                thr = float(threshold)
-                scored = [s for s in scored if s[0] >= thr]
-            except Exception:
-                pass
-
-        scored.sort(key=lambda x: (-x[0], x[1]))
-        return [u for _, u in scored]
-            
-    async def _extract_main_content(self, soup: BeautifulSoup, url: str) -> Dict[str, str]:
-        """
-        Extract main content using multiple strategies.
-        
-        Returns dict with 'title' and 'text' keys.
-        """
-        # Strategy 1: Try Readability algorithm (if available)
-        # Uncomment if readability-lxml is installed
-        # try:
-        #     doc = Readability(str(soup))
-        #     summary = doc.summary()
-        #     summary_soup = BeautifulSoup(summary, 'lxml')
-        #     
-        #     return {
-        #         'title': doc.title() or self._extract_title(soup),
-        #         'text': sanitize_text(summary_soup.get_text())
-        #     }
-        # except:
-        #     pass
-            
-        # Strategy 2: Look for common content containers
-        content_selectors = [
-            'main',
-            'article',
-            '[role="main"]',
-            '#content',
-            '.content',
-            '#main',
-            '.main',
-            'div.post',
-            'div.entry-content',
-            'div.article-body',
-            'div.story-body'
-        ]
-        
-        for selector in content_selectors:
-            content = soup.select_one(selector)
-            if content:
-                text = sanitize_text(content.get_text())
-                if len(text) > 100:  # Minimum content length
-                    return {
-                        'title': self._extract_title(soup),
-                        'text': text
-                    }
-                    
-        # Strategy 3: Find largest text block
-        text_blocks = []
-        for elem in soup.find_all(['div', 'section', 'article']):
-            text = sanitize_text(elem.get_text())
-            if len(text) > 50:
-                text_blocks.append((len(text), text, elem))
-                
-        if text_blocks:
-            text_blocks.sort(reverse=True)
-            return {
-                'title': self._extract_title(soup),
-                'text': text_blocks[0][1]
-            }
-            
-        # Fallback: Get all text
-        return {
-            'title': self._extract_title(soup),
-            'text': sanitize_text(soup.get_text())
-        }
-        
-    async def _extract_with_selectors(
-        self, 
-        soup: BeautifulSoup, 
-        selectors: Dict[str, str]
-    ) -> Dict[str, str]:
-        """Extract content using custom CSS selectors."""
-        result = {
-            'title': '',
-            'text': ''
-        }
-        
-        # Extract title
-        if 'title' in selectors:
-            title_elem = soup.select_one(selectors['title'])
-            if title_elem:
-                result['title'] = sanitize_text(title_elem.get_text())
-        else:
-            result['title'] = self._extract_title(soup)
-            
-        # Extract main content
-        if 'content' in selectors:
-            content_elem = soup.select_one(selectors['content'])
-            if content_elem:
-                result['text'] = sanitize_text(content_elem.get_text())
-        
-        # Extract additional fields
-        text_parts = []
-        for field, selector in selectors.items():
-            if field not in ['title', 'content']:
-                elems = soup.select(selector)
-                for elem in elems:
-                    text = sanitize_text(elem.get_text())
-                    if text:
-                        text_parts.append(text)
-                        
-        # Combine all text
-        if text_parts:
-            if result['text']:
-                result['text'] += '\n\n' + '\n'.join(text_parts)
-            else:
-                result['text'] = '\n'.join(text_parts)
-                
-        return result
-        
-    def _extract_title(self, soup: BeautifulSoup) -> str:
-        """Extract page title using multiple strategies."""
-        # Try standard title tag
-        title_tag = soup.find('title')
-        if title_tag:
-            return sanitize_text(title_tag.get_text())
-            
-        # Try meta property
-        meta_title = soup.find('meta', property='og:title')
-        if meta_title and meta_title.get('content'):
-            return sanitize_text(meta_title['content'])
-            
-        # Try h1 tag
-        h1_tag = soup.find('h1')
-        if h1_tag:
-            return sanitize_text(h1_tag.get_text())
-            
-        return "Untitled"
-        
-    async def extract_metadata(self, soup: BeautifulSoup, url: str) -> ContentMetadata:
-        """Extract structured metadata from the page."""
-        metadata = ContentMetadata()
-        
-        # Extract title
-        metadata.title = self._extract_title(soup)
-        
-        # Extract description
-        meta_desc = soup.find('meta', attrs={'name': 'description'})
-        if meta_desc and meta_desc.get('content'):
-            metadata.description = sanitize_text(meta_desc['content'])
-            
-        # Extract author
-        author_meta = soup.find('meta', attrs={'name': 'author'})
-        if author_meta and author_meta.get('content'):
-            metadata.author = sanitize_text(author_meta['content'])
-            
-        # Extract dates
-        date_published = soup.find('meta', property='article:published_time')
-        if date_published and date_published.get('content'):
-            try:
-                metadata.published_date = datetime.fromisoformat(date_published['content'].replace('Z', '+00:00'))
-            except:
-                pass
-                
-        # Extract keywords
-        keywords_meta = soup.find('meta', attrs={'name': 'keywords'})
-        if keywords_meta and keywords_meta.get('content'):
-            metadata.keywords = [k.strip() for k in keywords_meta['content'].split(',')]
-            
-        # Extract Open Graph data
-        for meta in soup.find_all('meta', property=re.compile('^og:')):
-            prop = meta.get('property', '').replace('og:', '')
-            content = meta.get('content', '')
-            if prop and content:
-                metadata.og_data[prop] = content
-                
-        # Extract Twitter Card data
-        for meta in soup.find_all('meta', attrs={'name': re.compile('^twitter:')}):
-            name = meta.get('name', '').replace('twitter:', '')
-            content = meta.get('content', '')
-            if name and content:
-                metadata.twitter_data[name] = content
-                
-        # Extract JSON-LD structured data
-        json_ld_scripts = soup.find_all('script', type='application/ld+json')
-        for script in json_ld_scripts:
-            try:
-                data = json.loads(script.string)
-                if isinstance(data, dict):
-                    metadata.json_ld = data
-                    break
-            except:
-                pass
-                
-        return metadata
-        
-    def _extract_images(self, soup: BeautifulSoup, base_url: str) -> List[str]:
-        """Extract all images from the page."""
-        images = []
-        seen = set()
-        
-        for img in soup.find_all(['img', 'picture']):
-            # Try different attributes
-            src = img.get('src') or img.get('data-src') or img.get('data-lazy-src')
-            
-            if not src:
-                # Check source tags within picture elements
-                if img.name == 'picture':
-                    source = img.find('source')
-                    if source:
-                        src = source.get('srcset', '').split()[0]
-                        
-            if src:
-                # Make URL absolute
-                abs_url = urljoin(base_url, src)
-                
-                # Skip if already seen or too small (likely tracking pixels)
-                if abs_url not in seen and not self._is_tracking_pixel(abs_url, img):
-                    seen.add(abs_url)
-                    images.append(abs_url)
-                    
-        return images[:100]  # Limit to 100 images
-        
-    def _extract_links(self, soup: BeautifulSoup, base_url: str) -> List[str]:
-        """Extract all links from the page."""
-        links = []
-        seen = set()
-        
-        for link in soup.find_all('a', href=True):
-            href = link['href']
-            
-            # Make URL absolute
-            abs_url = urljoin(base_url, href)
-            
-            # Skip anchors, javascript, and mail links
-            if (abs_url not in seen and 
-                not href.startswith(('#', 'javascript:', 'mailto:'))):
-                seen.add(abs_url)
-                links.append(abs_url)
-                
-        return links[:200]  # Limit to 200 links
-        
-    def _is_tracking_pixel(self, url: str, img_tag) -> bool:
-        """Check if an image is likely a tracking pixel."""
-        # Check dimensions
-        width = img_tag.get('width', '').replace('px', '')
-        height = img_tag.get('height', '').replace('px', '')
-        
-        try:
-            if width and height:
-                w, h = int(width), int(height)
-                if w <= 3 or h <= 3:
-                    return True
-        except:
-            pass
-            
-        # Check common tracking domains
-        tracking_domains = [
-            'google-analytics.com',
-            'googletagmanager.com',
-            'facebook.com/tr',
-            'doubleclick.net',
-            'scorecardresearch.com',
-            'quantserve.com',
-            'amazon-adsystem.com'
-        ]
-        
-        return any(domain in url for domain in tracking_domains)
-        
-    async def _check_robots_txt(self, url: str) -> bool:
-        """Check if URL is allowed by robots.txt."""
-        parsed = urlparse(url)
-        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
-        
-        # Check cache
-        if robots_url in self._robots_cache:
-            robot_parser = self._robots_cache[robots_url]
-            return robot_parser.can_fetch(self.user_agent, url)
-            
-        # Fetch and parse robots.txt
-        try:
-            robot_parser = RobotFileParser()
-            robot_parser.set_url(robots_url)
-            
-            # Fetch robots.txt content
-            response = await self._client.get(robots_url, timeout=5)
-            if response.status_code == 200:
-                robot_parser.parse(response.text.splitlines())
-            else:
-                # No robots.txt, allow all
-                robot_parser.allow_all = True
-                
-            # Cache parser
-            self._robots_cache[robots_url] = robot_parser
-            
-            return robot_parser.can_fetch(self.user_agent, url)
-            
-        except Exception as e:
-            logger.warning("robots_txt_check_failed", url=robots_url, error=str(e))
-            # On error, allow scraping
-            return True
-            
-    def _detect_encoding(self, response: httpx.Response) -> str:
-        """Detect response encoding using multiple methods."""
-        # Try charset from Content-Type header
-        content_type = response.headers.get('content-type', '')
-        match = re.search(r'charset=([^;]+)', content_type)
-        if match:
-            return match.group(1).strip()
-            
-        # Try to detect from content
-        detected = chardet.detect(response.content)
-        if detected['encoding'] and detected['confidence'] > 0.7:
-            return detected['encoding']
-            
-        # Default to UTF-8
-        return 'utf-8'
-
-
-# Singleton instance
-_scraping_service: Optional[ContentScrapingService] = None
-
-
-async def get_scraping_service() -> ContentScrapingService:
-    """Get or create scraping service instance."""
-    global _scraping_service
-    
-    if _scraping_service is None:
-        _scraping_service = ContentScrapingService()
-        await _scraping_service.initialize()
-        
-    return _scraping_service
diff --git a/apps/backend/app/services/searxng.py b/apps/backend/app/services/searxng.py
deleted file mode 100644
index a198cdd..0000000
--- a/apps/backend/app/services/searxng.py
+++ /dev/null
@@ -1,395 +0,0 @@
-"""
-SearXNG integration service for search operations.
-"""
-import asyncio
-import hashlib
-import json
-from typing import Dict, List, Optional, Any
-from urllib.parse import urlencode, urljoin
-import httpx
-from httpx import AsyncClient, HTTPError, TimeoutException
-from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
-
-from app.config import get_settings
-from app.models.responses import SearchResult, EngineInfo, ServiceHealth
-from app.utils.text_processing import sanitize_text, extract_snippet
-import structlog
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class SearXNGService:
-    """Service for interacting with SearXNG instance."""
-    
-    def __init__(self):
-        self.base_url = str(settings.searxng_url).rstrip('/')
-        self.timeout = settings.searxng_timeout
-        self.max_retries = settings.searxng_max_retries
-        self._client: Optional[AsyncClient] = None
-        self._session_cookies: Optional[Dict[str, str]] = None
-        
-    async def __aenter__(self):
-        """Async context manager entry."""
-        await self.initialize()
-        return self
-        
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        await self.close()
-        
-    async def initialize(self):
-        """Initialize HTTP client with connection pooling."""
-        if not self._client:
-            self._client = AsyncClient(
-                timeout=httpx.Timeout(self.timeout),
-                limits=httpx.Limits(
-                    max_keepalive_connections=10,
-                    max_connections=20,
-                    keepalive_expiry=30
-                ),
-                headers={
-                    "User-Agent": settings.scraping_user_agent,
-                    "Accept": "application/json",
-                    "Accept-Language": "en-US,en;q=0.9"
-                }
-            )
-            # Get initial session cookies
-            await self._get_session()
-            
-    async def close(self):
-        """Close HTTP client connections."""
-        if self._client:
-            await self._client.aclose()
-            self._client = None
-            
-    async def _get_session(self):
-        """Get SearXNG session cookies for proper operation."""
-        try:
-            response = await self._client.get(self.base_url)
-            self._session_cookies = dict(response.cookies)
-            logger.info("searxng_session_initialized", cookies_count=len(self._session_cookies))
-        except Exception as e:
-            logger.warning("searxng_session_init_failed", error=str(e))
-            
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=2, max=10),
-        retry=retry_if_exception_type((HTTPError, TimeoutException))
-    )
-    async def search(
-        self, 
-        query: str, 
-        engines: List[str], 
-        language: str = "en",
-        safe_search: int = 1,
-        time_range: Optional[str] = None,
-        categories: Optional[List[str]] = None,
-        pageno: int = 1,
-        **kwargs
-    ) -> List[SearchResult]:
-        """
-        Perform search using SearXNG.
-        
-        Args:
-            query: Search query
-            engines: List of search engines to use
-            language: Language code (e.g., 'en', 'de', 'fr')
-            safe_search: Safe search level (0=off, 1=moderate, 2=strict)
-            time_range: Time range filter (e.g., 'day', 'week', 'month', 'year')
-            categories: Search categories (e.g., ['general', 'images', 'news'])
-            pageno: Page number for pagination
-            
-        Returns:
-            List of SearchResult objects
-        """
-        if not self._client:
-            await self.initialize()
-            
-        # Build search parameters
-        params = {
-            "q": query,
-            "format": "json",
-            "language": language,
-            "safesearch": safe_search,
-            "pageno": pageno,
-        }
-        
-        # Add engines
-        if engines:
-            params["engines"] = ",".join(engines)
-            
-        # Add time range
-        if time_range and time_range in ["day", "week", "month", "year"]:
-            params["time_range"] = time_range
-            
-        # Add categories
-        if categories:
-            params["categories"] = ",".join(categories)
-            
-        # Merge additional parameters
-        params.update(kwargs)
-        
-        search_url = urljoin(self.base_url, "/search")
-        
-        try:
-            logger.info(
-                "searxng_search_start",
-                query=query,
-                engines=engines,
-                language=language,
-                params=params
-            )
-            
-            response = await self._client.get(
-                search_url,
-                params=params,
-                cookies=self._session_cookies
-            )
-            response.raise_for_status()
-            
-            data = response.json()
-            results = await self._parse_results(data, query)
-            
-            logger.info(
-                "searxng_search_completed",
-                query=query,
-                results_count=len(results),
-                response_time_ms=int(response.elapsed.total_seconds() * 1000)
-            )
-            
-            return results
-            
-        except httpx.HTTPStatusError as e:
-            logger.error(
-                "searxng_search_http_error",
-                query=query,
-                status_code=e.response.status_code,
-                error=str(e)
-            )
-            raise
-            
-        except Exception as e:
-            logger.error(
-                "searxng_search_error",
-                query=query,
-                error=str(e),
-                error_type=type(e).__name__
-            )
-            raise
-            
-    async def _parse_results(self, data: Dict[str, Any], query: str) -> List[SearchResult]:
-        """Parse SearXNG response into SearchResult objects."""
-        results = []
-        seen_urls = set()
-        
-        for idx, item in enumerate(data.get("results", [])):
-            url = item.get("url", "")
-            
-            # Skip duplicate URLs
-            if url in seen_urls:
-                continue
-            seen_urls.add(url)
-            
-            # Extract and sanitize fields
-            title = sanitize_text(item.get("title", ""))
-            content = item.get("content", "")
-            
-            # Generate snippet if content is too long
-            if len(content) > 300:
-                snippet = extract_snippet(content, query, max_length=250)
-            else:
-                snippet = sanitize_text(content)
-                
-            # Determine which engine returned this result
-            engine = item.get("engine", "unknown")
-            if isinstance(engine, list):
-                engine = engine[0] if engine else "unknown"
-                
-            result = SearchResult(
-                rank=idx + 1,
-                title=title or "Untitled",
-                url=url,
-                snippet=snippet,
-                engine=engine,
-                score=item.get("score"),
-                cached=False
-            )
-            
-            results.append(result)
-            
-        return results
-        
-    async def get_available_engines(self) -> Dict[str, EngineInfo]:
-        """Get list of available search engines with their capabilities."""
-        if not self._client:
-            await self.initialize()
-            
-        try:
-            # Get engine stats from SearXNG
-            config_url = urljoin(self.base_url, "/config")
-            response = await self._client.get(config_url)
-            response.raise_for_status()
-            
-            config_data = response.json()
-            engines_data = config_data.get("engines", [])
-            
-            engines = {}
-            for engine in engines_data:
-                name = engine.get("name", "")
-                if not name:
-                    continue
-                    
-                engine_info = EngineInfo(
-                    name=name,
-                    enabled=not engine.get("disabled", False),
-                    categories=engine.get("categories", ["general"]),
-                    supported_languages=engine.get("supported_languages", ["*"]),
-                    safe_search_support=engine.get("safesearch", False),
-                    time_range_support=engine.get("time_range_support", False),
-                    paging_support=engine.get("paging", False)
-                )
-                
-                engines[name] = engine_info
-                
-            logger.info(
-                "searxng_engines_fetched",
-                total_engines=len(engines),
-                enabled_engines=sum(1 for e in engines.values() if e.enabled)
-            )
-            
-            return engines
-            
-        except Exception as e:
-            logger.error("searxng_engines_fetch_error", error=str(e))
-            # Return default engines if fetch fails
-            return self._get_default_engines()
-            
-    def _get_default_engines(self) -> Dict[str, EngineInfo]:
-        """Get default engine configuration."""
-        default_engines = {
-            "google": EngineInfo(
-                name="google",
-                enabled=True,
-                categories=["general", "images", "news"],
-                supported_languages=["*"],
-                safe_search_support=True,
-                time_range_support=True,
-                paging_support=True
-            ),
-            "bing": EngineInfo(
-                name="bing",
-                enabled=True,
-                categories=["general", "images", "news"],
-                supported_languages=["*"],
-                safe_search_support=True,
-                time_range_support=True,
-                paging_support=True
-            ),
-            "duckduckgo": EngineInfo(
-                name="duckduckgo",
-                enabled=True,
-                categories=["general", "images"],
-                supported_languages=["*"],
-                safe_search_support=True,
-                time_range_support=True,
-                paging_support=True
-            ),
-            "startpage": EngineInfo(
-                name="startpage",
-                enabled=True,
-                categories=["general"],
-                supported_languages=["*"],
-                safe_search_support=True,
-                time_range_support=False,
-                paging_support=True
-            ),
-            "qwant": EngineInfo(
-                name="qwant",
-                enabled=True,
-                categories=["general", "images", "news"],
-                supported_languages=["*"],
-                safe_search_support=True,
-                time_range_support=False,
-                paging_support=True
-            )
-        }
-        
-        return default_engines
-        
-    async def health_check(self) -> ServiceHealth:
-        """Perform comprehensive health check on SearXNG service."""
-        start_time = asyncio.get_event_loop().time()
-        
-        try:
-            if not self._client:
-                await self.initialize()
-                
-            # Check basic connectivity
-            response = await self._client.get(
-                self.base_url,
-                timeout=httpx.Timeout(5.0)
-            )
-            response.raise_for_status()
-            
-            # Check search functionality with minimal query
-            test_results = await self.search(
-                query="test",
-                engines=["duckduckgo"],
-                language="en"
-            )
-            
-            latency_ms = int((asyncio.get_event_loop().time() - start_time) * 1000)
-            
-            return ServiceHealth(
-                status="healthy",
-                latency_ms=latency_ms,
-                last_check=asyncio.get_event_loop().time(),
-                details={
-                    "version": response.headers.get("X-SearXNG-Version", "unknown"),
-                    "test_results_count": len(test_results)
-                }
-            )
-            
-        except Exception as e:
-            latency_ms = int((asyncio.get_event_loop().time() - start_time) * 1000)
-            
-            return ServiceHealth(
-                status="unhealthy",
-                latency_ms=latency_ms,
-                last_check=asyncio.get_event_loop().time(),
-                details={
-                    "error": str(e),
-                    "error_type": type(e).__name__
-                }
-            )
-            
-    def generate_cache_key(self, query: str, engines: List[str], **params) -> str:
-        """Generate deterministic cache key for search query."""
-        # Sort engines and params for consistency
-        sorted_engines = sorted(engines)
-        sorted_params = sorted(params.items())
-        
-        key_data = {
-            "query": query.lower().strip(),
-            "engines": sorted_engines,
-            "params": sorted_params
-        }
-        
-        key_string = json.dumps(key_data, sort_keys=True)
-        return hashlib.sha256(key_string.encode()).hexdigest()
-
-
-# Singleton instance
-_searxng_service: Optional[SearXNGService] = None
-
-
-async def get_searxng_service() -> SearXNGService:
-    """Get or create SearXNG service instance."""
-    global _searxng_service
-    
-    if _searxng_service is None:
-        _searxng_service = SearXNGService()
-        await _searxng_service.initialize()
-        
-    return _searxng_service
diff --git a/apps/backend/app/services/stripe_service.py b/apps/backend/app/services/stripe_service.py
deleted file mode 100644
index 206ad00..0000000
--- a/apps/backend/app/services/stripe_service.py
+++ /dev/null
@@ -1,587 +0,0 @@
-"""
-Stripe payment and subscription service.
-"""
-import stripe
-from typing import Optional, Dict, Any, List
-from datetime import datetime, timedelta
-import structlog
-
-from app.config import get_settings
-from app.models.users import (
-    User, Subscription, Plan, Invoice, WebhookEvent,
-    PlanType, SubscriptionStatus
-)
-from app.services.database import DatabaseService, get_database_service
-from fastapi import Depends
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-# Configure Stripe
-stripe.api_key = settings.stripe_secret_key
-
-
-class StripeService:
-    """Service for managing Stripe payments and subscriptions."""
-    
-    def __init__(self, db_service: DatabaseService):
-        self.db = db_service
-        self.webhook_secret = settings.stripe_webhook_secret
-        
-    async def create_customer(self, user: User) -> str:
-        """Create a Stripe customer for a user."""
-        try:
-            customer = stripe.Customer.create(
-                email=user.email,
-                name=user.full_name,
-                metadata={
-                    "user_id": str(user.id),
-                    "username": user.username or ""
-                }
-            )
-            
-            # Update user with Stripe customer ID
-            user.stripe_customer_id = customer.id
-            await self.db.update_user(user)
-            
-            logger.info("stripe_customer_created", 
-                       user_id=user.id, 
-                       customer_id=customer.id)
-            
-            return customer.id
-            
-        except stripe.error.StripeError as e:
-            logger.error("stripe_customer_creation_failed", 
-                        user_id=user.id, 
-                        error=str(e))
-            raise
-    
-    async def create_subscription(
-        self, 
-        user: User, 
-        price_id: str,
-        trial_days: int = 0
-    ) -> Subscription:
-        """Create a subscription for a user."""
-        # Ensure customer exists
-        if not user.stripe_customer_id:
-            await self.create_customer(user)
-        
-        try:
-            # Create Stripe subscription
-            stripe_sub = stripe.Subscription.create(
-                customer=user.stripe_customer_id,
-                items=[{"price": price_id}],
-                trial_period_days=trial_days,
-                payment_behavior="default_incomplete",
-                expand=["latest_invoice.payment_intent"],
-                metadata={
-                    "user_id": str(user.id)
-                }
-            )
-            
-            # Get plan details
-            plan = await self.db.get_plan_by_price_id(price_id)
-            
-            # Create local subscription record
-            subscription = Subscription(
-                user_id=user.id,
-                stripe_subscription_id=stripe_sub.id,
-                stripe_price_id=price_id,
-                stripe_product_id=stripe_sub.items.data[0].price.product,
-                plan_type=self._get_plan_type(plan),
-                status=self._map_subscription_status(stripe_sub.status),
-                amount=stripe_sub.items.data[0].price.unit_amount / 100,
-                currency=stripe_sub.currency,
-                interval=stripe_sub.items.data[0].price.recurring.interval,
-                search_limit=plan.search_limit if plan else 1000,
-                scrape_limit=plan.scrape_limit if plan else 10000,
-                rate_limit=plan.rate_limit if plan else "100/hour",
-                features=plan.features if plan else {},
-                trial_start=datetime.fromtimestamp(stripe_sub.trial_start) if stripe_sub.trial_start else None,
-                trial_end=datetime.fromtimestamp(stripe_sub.trial_end) if stripe_sub.trial_end else None,
-                current_period_start=datetime.fromtimestamp(stripe_sub.current_period_start),
-                current_period_end=datetime.fromtimestamp(stripe_sub.current_period_end)
-            )
-            
-            await self.db.create_subscription(subscription)
-            
-            logger.info("subscription_created",
-                       user_id=user.id,
-                       subscription_id=stripe_sub.id,
-                       plan=plan.name if plan else "unknown")
-            
-            return subscription
-            
-        except stripe.error.StripeError as e:
-            logger.error("subscription_creation_failed",
-                        user_id=user.id,
-                        error=str(e))
-            raise
-    
-    async def cancel_subscription(self, subscription: Subscription, immediately: bool = False) -> Subscription:
-        """Cancel a subscription."""
-        try:
-            if immediately:
-                # Cancel immediately
-                stripe_sub = stripe.Subscription.delete(subscription.stripe_subscription_id)
-            else:
-                # Cancel at period end
-                stripe_sub = stripe.Subscription.modify(
-                    subscription.stripe_subscription_id,
-                    cancel_at_period_end=True
-                )
-            
-            # Update local subscription
-            subscription.status = SubscriptionStatus.CANCELLED
-            subscription.cancelled_at = datetime.utcnow()
-            if immediately:
-                subscription.ended_at = datetime.utcnow()
-            
-            await self.db.update_subscription(subscription)
-            
-            logger.info("subscription_cancelled",
-                       subscription_id=subscription.id,
-                       immediately=immediately)
-            
-            return subscription
-            
-        except stripe.error.StripeError as e:
-            logger.error("subscription_cancellation_failed",
-                        subscription_id=subscription.id,
-                        error=str(e))
-            raise
-    
-    async def update_subscription(self, subscription: Subscription, new_price_id: str) -> Subscription:
-        """Update subscription to a different plan."""
-        try:
-            # Get the subscription item ID
-            stripe_sub = stripe.Subscription.retrieve(subscription.stripe_subscription_id)
-            item_id = stripe_sub.items.data[0].id
-            
-            # Update the subscription
-            stripe_sub = stripe.Subscription.modify(
-                subscription.stripe_subscription_id,
-                items=[{
-                    "id": item_id,
-                    "price": new_price_id
-                }],
-                proration_behavior="create_prorations"
-            )
-            
-            # Get new plan details
-            plan = await self.db.get_plan_by_price_id(new_price_id)
-            
-            # Update local subscription
-            subscription.stripe_price_id = new_price_id
-            subscription.plan_type = self._get_plan_type(plan)
-            subscription.amount = stripe_sub.items.data[0].price.unit_amount / 100
-            subscription.search_limit = plan.search_limit if plan else None
-            subscription.scrape_limit = plan.scrape_limit if plan else None
-            subscription.rate_limit = plan.rate_limit if plan else "1000/hour"
-            subscription.features = plan.features if plan else {}
-            
-            await self.db.update_subscription(subscription)
-            
-            logger.info("subscription_updated",
-                       subscription_id=subscription.id,
-                       new_plan=plan.name if plan else "unknown")
-            
-            return subscription
-            
-        except stripe.error.StripeError as e:
-            logger.error("subscription_update_failed",
-                        subscription_id=subscription.id,
-                        error=str(e))
-            raise
-    
-    async def create_payment_intent(
-        self,
-        user: User,
-        amount: int,
-        currency: str = "usd",
-        description: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """Create a payment intent for one-time payment."""
-        if not user.stripe_customer_id:
-            await self.create_customer(user)
-        
-        try:
-            intent = stripe.PaymentIntent.create(
-                amount=amount,  # Amount in cents
-                currency=currency,
-                customer=user.stripe_customer_id,
-                description=description,
-                metadata={
-                    "user_id": str(user.id)
-                }
-            )
-            
-            return {
-                "client_secret": intent.client_secret,
-                "payment_intent_id": intent.id
-            }
-            
-        except stripe.error.StripeError as e:
-            logger.error("payment_intent_creation_failed",
-                        user_id=user.id,
-                        error=str(e))
-            raise
-    
-    async def create_checkout_session(
-        self,
-        user: User,
-        price_id: str,
-        success_url: str,
-        cancel_url: str,
-        trial_days: int = 0
-    ) -> str:
-        """Create a Stripe Checkout session."""
-        if not user.stripe_customer_id:
-            await self.create_customer(user)
-        
-        try:
-            session = stripe.checkout.Session.create(
-                customer=user.stripe_customer_id,
-                payment_method_types=["card"],
-                line_items=[{
-                    "price": price_id,
-                    "quantity": 1
-                }],
-                mode="subscription",
-                success_url=success_url,
-                cancel_url=cancel_url,
-                subscription_data={
-                    "trial_period_days": trial_days,
-                    "metadata": {
-                        "user_id": str(user.id)
-                    }
-                }
-            )
-            
-            logger.info("checkout_session_created",
-                       user_id=user.id,
-                       session_id=session.id)
-            
-            return session.url
-            
-        except stripe.error.StripeError as e:
-            logger.error("checkout_session_creation_failed",
-                        user_id=user.id,
-                        error=str(e))
-            raise
-    
-    async def create_billing_portal_session(self, user: User, return_url: str) -> str:
-        """Create a billing portal session for subscription management."""
-        if not user.stripe_customer_id:
-            raise ValueError("User has no Stripe customer ID")
-        
-        try:
-            session = stripe.billing_portal.Session.create(
-                customer=user.stripe_customer_id,
-                return_url=return_url
-            )
-            
-            return session.url
-            
-        except stripe.error.StripeError as e:
-            logger.error("billing_portal_session_creation_failed",
-                        user_id=user.id,
-                        error=str(e))
-            raise
-    
-    async def handle_webhook(self, payload: str, signature: str) -> bool:
-        """Handle Stripe webhook events."""
-        try:
-            # Verify webhook signature
-            event = stripe.Webhook.construct_event(
-                payload, signature, self.webhook_secret
-            )
-            
-            # Check if we've already processed this event
-            existing = await self.db.get_webhook_event(event.id)
-            if existing and existing.processed:
-                logger.info("webhook_already_processed", event_id=event.id)
-                return True
-            
-            # Store webhook event
-            webhook_event = await self.db.create_webhook_event(
-                stripe_event_id=event.id,
-                event_type=event.type,
-                data=event.data.object
-            )
-            
-            # Process event based on type
-            if event.type == "customer.subscription.created":
-                await self._handle_subscription_created(event.data.object)
-            elif event.type == "customer.subscription.updated":
-                await self._handle_subscription_updated(event.data.object)
-            elif event.type == "customer.subscription.deleted":
-                await self._handle_subscription_deleted(event.data.object)
-            elif event.type == "invoice.paid":
-                await self._handle_invoice_paid(event.data.object)
-            elif event.type == "invoice.payment_failed":
-                await self._handle_invoice_payment_failed(event.data.object)
-            elif event.type == "payment_intent.succeeded":
-                await self._handle_payment_succeeded(event.data.object)
-            
-            # Mark webhook as processed
-            webhook_event.processed = True
-            webhook_event.processed_at = datetime.utcnow()
-            await self.db.update_webhook_event(webhook_event)
-            
-            logger.info("webhook_processed",
-                       event_id=event.id,
-                       event_type=event.type)
-            
-            return True
-            
-        except stripe.error.SignatureVerificationError:
-            logger.error("webhook_signature_verification_failed")
-            return False
-        except Exception as e:
-            logger.error("webhook_processing_failed", error=str(e))
-            return False
-    
-    async def _handle_subscription_created(self, stripe_sub):
-        """Handle subscription created event."""
-        user = await self.db.get_user_by_stripe_customer(stripe_sub.customer)
-        if not user:
-            logger.error("user_not_found_for_subscription", customer_id=stripe_sub.customer)
-            return
-        
-        # Check if subscription already exists
-        subscription = await self.db.get_subscription_by_stripe_id(stripe_sub.id)
-        if subscription:
-            return
-        
-        # Get plan details
-        price_id = stripe_sub.items.data[0].price.id
-        plan = await self.db.get_plan_by_price_id(price_id)
-        
-        # Create subscription record
-        subscription = Subscription(
-            user_id=user.id,
-            stripe_subscription_id=stripe_sub.id,
-            stripe_price_id=price_id,
-            stripe_product_id=stripe_sub.items.data[0].price.product,
-            plan_type=self._get_plan_type(plan),
-            status=self._map_subscription_status(stripe_sub.status),
-            amount=stripe_sub.items.data[0].price.unit_amount / 100,
-            currency=stripe_sub.currency,
-            interval=stripe_sub.items.data[0].price.recurring.interval,
-            search_limit=plan.search_limit if plan else 1000,
-            scrape_limit=plan.scrape_limit if plan else 10000,
-            rate_limit=plan.rate_limit if plan else "100/hour",
-            features=plan.features if plan else {},
-            trial_start=datetime.fromtimestamp(stripe_sub.trial_start) if stripe_sub.trial_start else None,
-            trial_end=datetime.fromtimestamp(stripe_sub.trial_end) if stripe_sub.trial_end else None,
-            current_period_start=datetime.fromtimestamp(stripe_sub.current_period_start),
-            current_period_end=datetime.fromtimestamp(stripe_sub.current_period_end)
-        )
-        
-        await self.db.create_subscription(subscription)
-        
-    async def _handle_subscription_updated(self, stripe_sub):
-        """Handle subscription updated event."""
-        subscription = await self.db.get_subscription_by_stripe_id(stripe_sub.id)
-        if not subscription:
-            logger.error("subscription_not_found", stripe_id=stripe_sub.id)
-            return
-        
-        # Update subscription details
-        subscription.status = self._map_subscription_status(stripe_sub.status)
-        subscription.current_period_start = datetime.fromtimestamp(stripe_sub.current_period_start)
-        subscription.current_period_end = datetime.fromtimestamp(stripe_sub.current_period_end)
-        
-        if stripe_sub.cancel_at_period_end:
-            subscription.status = SubscriptionStatus.CANCELLED
-        
-        await self.db.update_subscription(subscription)
-    
-    async def _handle_subscription_deleted(self, stripe_sub):
-        """Handle subscription deleted event."""
-        subscription = await self.db.get_subscription_by_stripe_id(stripe_sub.id)
-        if not subscription:
-            return
-        
-        subscription.status = SubscriptionStatus.CANCELLED
-        subscription.ended_at = datetime.utcnow()
-        await self.db.update_subscription(subscription)
-    
-    async def _handle_invoice_paid(self, invoice):
-        """Handle invoice paid event."""
-        user = await self.db.get_user_by_stripe_customer(invoice.customer)
-        if not user:
-            return
-        
-        # Create or update invoice record
-        inv = await self.db.get_invoice_by_stripe_id(invoice.id)
-        if not inv:
-            inv = Invoice(
-                user_id=user.id,
-                stripe_invoice_id=invoice.id,
-                stripe_charge_id=invoice.charge,
-                invoice_number=invoice.number,
-                status="paid",
-                amount_due=invoice.amount_due,
-                amount_paid=invoice.amount_paid,
-                amount_remaining=invoice.amount_remaining,
-                subtotal=invoice.subtotal,
-                tax=invoice.tax,
-                total=invoice.total,
-                currency=invoice.currency,
-                period_start=datetime.fromtimestamp(invoice.period_start) if invoice.period_start else None,
-                period_end=datetime.fromtimestamp(invoice.period_end) if invoice.period_end else None,
-                paid_at=datetime.fromtimestamp(invoice.status_transitions.paid_at) if invoice.status_transitions.paid_at else None,
-                invoice_pdf=invoice.invoice_pdf,
-                hosted_invoice_url=invoice.hosted_invoice_url
-            )
-            await self.db.create_invoice(inv)
-        else:
-            inv.status = "paid"
-            inv.paid_at = datetime.fromtimestamp(invoice.status_transitions.paid_at) if invoice.status_transitions.paid_at else None
-            await self.db.update_invoice(inv)
-        
-        # Reset usage for new billing period
-        await self.db.reset_user_usage(user.id)
-    
-    async def _handle_invoice_payment_failed(self, invoice):
-        """Handle invoice payment failed event."""
-        subscription = await self.db.get_subscription_by_stripe_customer(invoice.customer)
-        if subscription:
-            subscription.status = SubscriptionStatus.PAST_DUE
-            await self.db.update_subscription(subscription)
-    
-    async def _handle_payment_succeeded(self, payment_intent):
-        """Handle payment intent succeeded event."""
-        logger.info("payment_succeeded",
-                   payment_intent_id=payment_intent.id,
-                   amount=payment_intent.amount)
-    
-    def _map_subscription_status(self, stripe_status: str) -> SubscriptionStatus:
-        """Map Stripe subscription status to our enum."""
-        mapping = {
-            "active": SubscriptionStatus.ACTIVE,
-            "trialing": SubscriptionStatus.TRIALING,
-            "canceled": SubscriptionStatus.CANCELLED,
-            "past_due": SubscriptionStatus.PAST_DUE,
-            "unpaid": SubscriptionStatus.UNPAID,
-            "incomplete": SubscriptionStatus.INCOMPLETE
-        }
-        return mapping.get(stripe_status, SubscriptionStatus.CANCELLED)
-    
-    def _get_plan_type(self, plan: Optional[Plan]) -> PlanType:
-        """Get plan type from plan object."""
-        if not plan:
-            return PlanType.FREE
-        
-        if "pro" in plan.name.lower():
-            return PlanType.PRO
-        elif "enterprise" in plan.name.lower():
-            return PlanType.ENTERPRISE
-        else:
-            return PlanType.FREE
-    
-    async def setup_default_plans(self):
-        """Set up default pricing plans in Stripe and database."""
-        plans_config = [
-            {
-                "name": "free",
-                "display_name": "Free Plan",
-                "description": "Perfect for getting started",
-                "price": 0,
-                "search_limit": 1000,
-                "scrape_limit": 10000,
-                "rate_limit": "100/hour",
-                "features": {
-                    "api_access": True,
-                    "webhook_support": False,
-                    "priority_support": False
-                }
-            },
-            {
-                "name": "pro",
-                "display_name": "Pro Plan",
-                "description": "Unlimited searches and scrapes",
-                "price": 20.00,
-                "search_limit": None,  # Unlimited
-                "scrape_limit": None,  # Unlimited
-                "rate_limit": "1000/hour",
-                "features": {
-                    "api_access": True,
-                    "webhook_support": True,
-                    "priority_support": True,
-                    "custom_engines": True
-                }
-            }
-        ]
-        
-        for plan_config in plans_config:
-            # Check if plan exists
-            existing = await self.db.get_plan_by_name(plan_config["name"])
-            if existing:
-                continue
-            
-            # Create Stripe product and price if not free
-            if plan_config["price"] > 0:
-                product = stripe.Product.create(
-                    name=plan_config["display_name"],
-                    description=plan_config["description"],
-                    metadata={
-                        "plan_name": plan_config["name"]
-                    }
-                )
-                
-                price = stripe.Price.create(
-                    product=product.id,
-                    unit_amount=int(plan_config["price"] * 100),  # Convert to cents
-                    currency="usd",
-                    recurring={"interval": "month"}
-                )
-                
-                stripe_product_id = product.id
-                stripe_price_id = price.id
-            else:
-                stripe_product_id = None
-                stripe_price_id = None
-            
-            # Create plan in database
-            plan = Plan(
-                name=plan_config["name"],
-                display_name=plan_config["display_name"],
-                description=plan_config["description"],
-                stripe_product_id=stripe_product_id,
-                stripe_price_id=stripe_price_id,
-                price=plan_config["price"],
-                currency="usd",
-                interval="month",
-                search_limit=plan_config["search_limit"],
-                scrape_limit=plan_config["scrape_limit"],
-                rate_limit=plan_config["rate_limit"],
-                features=plan_config["features"],
-                is_active=True,
-                is_visible=True
-            )
-            
-            await self.db.create_plan(plan)
-            
-            logger.info("plan_created",
-                       name=plan_config["name"],
-                       price=plan_config["price"])
-
-
-# Singleton instance
-_stripe_service: Optional[StripeService] = None
-
-
-async def get_stripe_service(db_service: DatabaseService = Depends(get_database_service)) -> StripeService:
-    """Get or create Stripe service instance."""
-    global _stripe_service
-    
-    if _stripe_service is None:
-        _stripe_service = StripeService(db_service)
-        # Set up default plans if needed
-        await _stripe_service.setup_default_plans()
-    
-    return _stripe_service
diff --git a/apps/backend/app/services/table_extraction.py b/apps/backend/app/services/table_extraction.py
deleted file mode 100644
index ed1cfd7..0000000
--- a/apps/backend/app/services/table_extraction.py
+++ /dev/null
@@ -1,751 +0,0 @@
-"""
-Table extraction strategies for detecting and extracting tables from HTML content.
-
-This module provides various strategies for table extraction:
-- DefaultTableExtraction: Score-based table detection and extraction
-- LLMTableExtraction: AI-powered table understanding and extraction
-- NoTableExtraction: Skip table extraction
-"""
-
-import json
-import re
-import time
-from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Any, Union, Tuple
-from urllib.parse import urljoin
-
-import structlog
-from bs4 import BeautifulSoup, Tag, NavigableString
-
-from app.utils.text_processing import sanitize_text
-
-logger = structlog.get_logger(__name__)
-
-
-class TableExtractionStrategy(ABC):
-    """
-    Abstract base class for all table extraction strategies.
-    
-    This class defines the interface that all table extraction strategies must implement.
-    """
-    
-    def __init__(self, **kwargs):
-        """
-        Initialize the table extraction strategy.
-        
-        Args:
-            **kwargs: Additional keyword arguments for specific strategies
-        """
-        self.verbose = kwargs.get("verbose", False)
-    
-    @abstractmethod
-    def extract_tables(self, html_content: str, base_url: str = "", **kwargs) -> List[Dict[str, Any]]:
-        """
-        Extract tables from the given HTML content.
-        
-        Args:
-            html_content: HTML content to extract tables from
-            base_url: Base URL for resolving relative links
-            **kwargs: Additional parameters for extraction
-            
-        Returns:
-            List of dictionaries containing table data, each with:
-                - headers: List of column headers
-                - rows: List of row data (each row is a list)
-                - caption: Table caption if present
-                - summary: Table summary attribute if present  
-                - metadata: Additional metadata about the table
-        """
-        pass
-
-
-class NoTableExtraction(TableExtractionStrategy):
-    """Table extraction strategy that skips table processing."""
-    
-    def extract_tables(self, html_content: str, base_url: str = "", **kwargs) -> List[Dict[str, Any]]:
-        """Return empty list - no table extraction."""
-        return []
-
-
-class DefaultTableExtraction(TableExtractionStrategy):
-    """
-    Default table extraction strategy using scoring to identify data tables.
-    
-    This strategy uses a scoring system to differentiate between layout tables
-    and actual data tables, then extracts structured data while handling
-    colspan and rowspan attributes.
-    """
-    
-    def __init__(self, **kwargs):
-        """
-        Initialize the default table extraction strategy.
-        
-        Args:
-            table_score_threshold (int): Minimum score for a table to be considered data table (default: 7)
-            min_rows (int): Minimum number of rows for a valid table (default: 2)
-            min_cols (int): Minimum number of columns for a valid table (default: 2)
-            extract_links (bool): Whether to extract links within table cells (default: True)
-            **kwargs: Additional parameters
-        """
-        super().__init__(**kwargs)
-        self.table_score_threshold = kwargs.get("table_score_threshold", 7)
-        self.min_rows = kwargs.get("min_rows", 2)
-        self.min_cols = kwargs.get("min_cols", 2)
-        self.extract_links = kwargs.get("extract_links", True)
-    
-    def extract_tables(self, html_content: str, base_url: str = "", **kwargs) -> List[Dict[str, Any]]:
-        """Extract all data tables from the HTML content."""
-        if not html_content:
-            return []
-        
-        soup = BeautifulSoup(html_content, 'lxml')
-        table_tags = soup.find_all('table')
-        
-        if not table_tags:
-            return []
-        
-        extracted_tables = []
-        
-        for idx, table_tag in enumerate(table_tags):
-            try:
-                # Score the table to determine if it's a data table
-                table_score = self._score_table(table_tag)
-                
-                if table_score < self.table_score_threshold:
-                    if self.verbose:
-                        logger.debug(f"Table {idx} skipped (score: {table_score} < {self.table_score_threshold})")
-                    continue
-                
-                # Extract table data
-                table_data = self._extract_table_data(table_tag, base_url, idx)
-                
-                if table_data and self._is_valid_table(table_data):
-                    table_data['metadata']['score'] = table_score
-                    extracted_tables.append(table_data)
-                    
-                    if self.verbose:
-                        logger.debug(f"Table {idx} extracted (score: {table_score}, rows: {len(table_data['rows'])})")
-                
-            except Exception as e:
-                logger.error(f"Error extracting table {idx}: {str(e)}")
-                continue
-        
-        logger.info(f"Extracted {len(extracted_tables)} tables from HTML content")
-        return extracted_tables
-    
-    def _score_table(self, table_tag: Tag) -> int:
-        """
-        Score a table to determine if it's likely a data table vs layout table.
-        
-        Args:
-            table_tag: BeautifulSoup table tag
-            
-        Returns:
-            Integer score (higher = more likely to be data table)
-        """
-        score = 0
-        
-        # Check for table headers
-        th_tags = table_tag.find_all('th')
-        if th_tags:
-            score += 5
-            # More headers = higher score
-            score += min(len(th_tags), 5)
-        
-        # Check for thead/tbody structure
-        if table_tag.find('thead'):
-            score += 3
-        if table_tag.find('tbody'):
-            score += 2
-        
-        # Check for caption
-        if table_tag.find('caption'):
-            score += 2
-        
-        # Check for summary attribute
-        if table_tag.get('summary'):
-            score += 1
-        
-        # Count rows and columns
-        rows = table_tag.find_all('tr')
-        if rows:
-            # More rows generally indicate data table
-            row_count = len(rows)
-            if row_count > 5:
-                score += 3
-            elif row_count > 2:
-                score += 1
-            
-            # Check column consistency
-            col_counts = []
-            for row in rows:
-                cells = row.find_all(['td', 'th'])
-                col_counts.append(len(cells))
-            
-            if col_counts:
-                # Consistent column count is good
-                if len(set(col_counts)) == 1:
-                    score += 2
-                
-                # More columns can indicate data table
-                max_cols = max(col_counts)
-                if max_cols > 4:
-                    score += 2
-                elif max_cols > 2:
-                    score += 1
-        
-        # Check for data-like attributes
-        data_attributes = ['data-table', 'data-grid', 'sortable']
-        for attr in data_attributes:
-            if table_tag.get(attr):
-                score += 1
-        
-        # Check class names for data table indicators
-        classes = table_tag.get('class', [])
-        data_class_indicators = ['data', 'grid', 'sortable', 'results', 'listing']
-        for indicator in data_class_indicators:
-            if any(indicator in str(cls).lower() for cls in classes):
-                score += 1
-        
-        # Penalize layout table indicators
-        layout_indicators = ['layout', 'wrapper', 'container', 'nav']
-        for indicator in layout_indicators:
-            if any(indicator in str(cls).lower() for cls in classes):
-                score -= 2
-        
-        # Check for form elements (usually layout tables)
-        if table_tag.find(['input', 'select', 'textarea']):
-            score -= 3
-        
-        return max(0, score)  # Ensure non-negative
-    
-    def _extract_table_data(self, table_tag: Tag, base_url: str, table_index: int) -> Dict[str, Any]:
-        """Extract structured data from a table."""
-        # Initialize table data structure
-        table_data = {
-            'headers': [],
-            'rows': [],
-            'caption': None,
-            'summary': None,
-            'metadata': {
-                'index': table_index,
-                'row_count': 0,
-                'col_count': 0,
-                'has_header': False,
-                'has_footer': False,
-                'extraction_time': time.time()
-            }
-        }
-        
-        # Extract caption
-        caption_tag = table_tag.find('caption')
-        if caption_tag:
-            table_data['caption'] = sanitize_text(caption_tag.get_text())
-        
-        # Extract summary
-        summary = table_tag.get('summary')
-        if summary:
-            table_data['summary'] = sanitize_text(summary)
-        
-        # Find all rows
-        rows = table_tag.find_all('tr')
-        if not rows:
-            return table_data
-        
-        # Process header row(s)
-        header_rows = []
-        data_rows = []
-        
-        # Check for thead section
-        thead = table_tag.find('thead')
-        if thead:
-            header_rows = thead.find_all('tr')
-            # Remaining rows are in tbody or directly in table
-            tbody = table_tag.find('tbody')
-            if tbody:
-                data_rows = tbody.find_all('tr')
-            else:
-                # Find rows not in thead
-                all_rows = table_tag.find_all('tr')
-                thead_rows = set(thead.find_all('tr'))
-                data_rows = [row for row in all_rows if row not in thead_rows]
-        else:
-            # No explicit thead, use heuristics
-            # Check if first row has mostly th tags
-            first_row = rows[0]
-            th_count = len(first_row.find_all('th'))
-            td_count = len(first_row.find_all('td'))
-            
-            if th_count > td_count:
-                header_rows = [first_row]
-                data_rows = rows[1:]
-            else:
-                data_rows = rows
-        
-        # Extract headers
-        if header_rows:
-            table_data['metadata']['has_header'] = True
-            for header_row in header_rows:
-                header_cells = header_row.find_all(['th', 'td'])
-                if not table_data['headers']:  # First header row
-                    table_data['headers'] = [
-                        sanitize_text(cell.get_text()) for cell in header_cells
-                    ]
-                # Note: Multi-row headers could be handled more sophisticatedly
-        
-        # Extract data rows
-        for row_idx, row in enumerate(data_rows):
-            cells = row.find_all(['td', 'th'])
-            row_data = []
-            
-            for cell in cells:
-                cell_data = self._extract_cell_data(cell, base_url)
-                
-                # Handle colspan
-                colspan = int(cell.get('colspan', 1))
-                if colspan > 1:
-                    # Add empty cells for colspan
-                    row_data.extend([cell_data] + [''] * (colspan - 1))
-                else:
-                    row_data.append(cell_data)
-            
-            if row_data:
-                table_data['rows'].append(row_data)
-        
-        # Update metadata
-        table_data['metadata']['row_count'] = len(table_data['rows'])
-        if table_data['rows']:
-            table_data['metadata']['col_count'] = max(len(row) for row in table_data['rows'])
-        
-        # Check for footer
-        tfoot = table_tag.find('tfoot')
-        if tfoot:
-            table_data['metadata']['has_footer'] = True
-        
-        return table_data
-    
-    def _extract_cell_data(self, cell: Tag, base_url: str) -> Union[str, Dict[str, Any]]:
-        """Extract data from a table cell, handling links and formatting."""
-        cell_text = sanitize_text(cell.get_text())
-        
-        # If extract_links is disabled, just return text
-        if not self.extract_links:
-            return cell_text
-        
-        # Check for links within the cell
-        links = cell.find_all('a', href=True)
-        if links:
-            cell_links = []
-            for link in links:
-                href = link.get('href')
-                if href:
-                    # Make URL absolute
-                    abs_url = urljoin(base_url, href) if base_url else href
-                    link_text = sanitize_text(link.get_text())
-                    cell_links.append({
-                        'text': link_text,
-                        'url': abs_url
-                    })
-            
-            if cell_links:
-                return {
-                    'text': cell_text,
-                    'links': cell_links
-                }
-        
-        return cell_text
-    
-    def _is_valid_table(self, table_data: Dict[str, Any]) -> bool:
-        """Check if extracted table data meets minimum requirements."""
-        row_count = len(table_data['rows'])
-        
-        if row_count < self.min_rows:
-            return False
-        
-        if table_data['rows']:
-            max_cols = max(len(row) for row in table_data['rows'])
-            if max_cols < self.min_cols:
-                return False
-        
-        # Check if table has meaningful content
-        total_chars = 0
-        for row in table_data['rows']:
-            for cell in row:
-                if isinstance(cell, str):
-                    total_chars += len(cell)
-                elif isinstance(cell, dict) and 'text' in cell:
-                    total_chars += len(cell['text'])
-        
-        # Table should have reasonable amount of text content
-        if total_chars < 10:
-            return False
-        
-        return True
-
-
-class LLMTableExtraction(TableExtractionStrategy):
-    """
-    AI-powered table extraction using language models.
-    
-    This strategy uses LLMs to understand and extract table content,
-    including complex tables that might be difficult for rule-based approaches.
-    """
-    
-    def __init__(self, llm_config: Optional[Dict[str, Any]] = None, **kwargs):
-        """
-        Initialize LLM table extraction strategy.
-        
-        Args:
-            llm_config: Configuration for LLM provider
-            **kwargs: Additional parameters
-        """
-        super().__init__(**kwargs)
-        self.llm_config = llm_config or {}
-        self.max_table_size = kwargs.get('max_table_size', 5000)  # Max chars per table for LLM
-        self.extraction_prompt_template = kwargs.get('extraction_prompt_template', self._default_prompt())
-    
-    def extract_tables(self, html_content: str, base_url: str = "", **kwargs) -> List[Dict[str, Any]]:
-        """Extract tables using LLM understanding."""
-        if not html_content:
-            return []
-        
-        soup = BeautifulSoup(html_content, 'lxml')
-        table_tags = soup.find_all('table')
-        
-        if not table_tags:
-            return []
-        
-        extracted_tables = []
-        
-        for idx, table_tag in enumerate(table_tags):
-            try:
-                # Convert table to text for LLM processing
-                table_html = str(table_tag)
-                
-                # Skip very large tables
-                if len(table_html) > self.max_table_size:
-                    if self.verbose:
-                        logger.debug(f"Table {idx} too large for LLM processing ({len(table_html)} chars)")
-                    continue
-                
-                # Use LLM to extract and structure table data
-                table_data = await self._extract_with_llm(table_html, idx, base_url)
-                
-                if table_data:
-                    extracted_tables.append(table_data)
-                    if self.verbose:
-                        logger.debug(f"Table {idx} extracted with LLM")
-            
-            except Exception as e:
-                logger.error(f"Error extracting table {idx} with LLM: {str(e)}")
-                continue
-        
-        logger.info(f"Extracted {len(extracted_tables)} tables using LLM")
-        return extracted_tables
-    
-    async def _extract_with_llm(self, table_html: str, table_index: int, base_url: str) -> Optional[Dict[str, Any]]:
-        """Use LLM to extract and structure table data."""
-        try:
-            # Prepare prompt
-            prompt = self.extraction_prompt_template.format(
-                table_html=table_html,
-                base_url=base_url or "not provided"
-            )
-            
-            # Mock LLM call - in production, integrate with actual LLM providers
-            # This would be replaced with actual LLM API calls
-            await asyncio.sleep(0.1)  # Simulate API delay
-            
-            # Mock structured response
-            mock_response = {
-                'headers': ['Column 1', 'Column 2', 'Column 3'],
-                'rows': [
-                    ['Row 1 Cell 1', 'Row 1 Cell 2', 'Row 1 Cell 3'],
-                    ['Row 2 Cell 1', 'Row 2 Cell 2', 'Row 2 Cell 3']
-                ],
-                'caption': 'Mock table extracted by LLM',
-                'summary': 'This is a mock extraction result',
-                'metadata': {
-                    'index': table_index,
-                    'extraction_method': 'llm',
-                    'row_count': 2,
-                    'col_count': 3,
-                    'confidence': 0.85,
-                    'extraction_time': time.time()
-                }
-            }
-            
-            return mock_response
-            
-        except Exception as e:
-            logger.error(f"LLM table extraction failed: {str(e)}")
-            return None
-    
-    def _default_prompt(self) -> str:
-        """Default prompt template for LLM table extraction."""
-        return """
-Extract and structure the following HTML table into JSON format.
-
-Instructions:
-1. Identify table headers (if any) and include them in the 'headers' array
-2. Extract all data rows into the 'rows' array (each row is an array of cell values)
-3. Include any caption or summary information
-4. Preserve the semantic meaning and structure of the table
-5. If there are links in cells, extract both the text and URL
-
-HTML Table:
-{table_html}
-
-Base URL (for resolving relative links): {base_url}
-
-Return the result as valid JSON with this structure:
-{{
-    "headers": ["header1", "header2", ...],
-    "rows": [["cell1", "cell2", ...], ["cell1", "cell2", ...], ...],
-    "caption": "table caption or null",
-    "summary": "table summary or null",
-    "metadata": {{
-        "extraction_method": "llm",
-        "confidence": 0.0-1.0,
-        "notes": "any relevant notes about the table"
-    }}
-}}
-"""
-
-
-class SmartTableExtraction(TableExtractionStrategy):
-    """
-    Smart table extraction that combines rule-based and AI approaches.
-    
-    Uses rule-based extraction for simple tables and falls back to
-    LLM extraction for complex cases.
-    """
-    
-    def __init__(self, **kwargs):
-        """Initialize smart table extraction strategy."""
-        super().__init__(**kwargs)
-        
-        # Initialize both strategies
-        self.default_extractor = DefaultTableExtraction(**kwargs)
-        self.llm_extractor = LLMTableExtraction(**kwargs)
-        
-        self.complexity_threshold = kwargs.get('complexity_threshold', 15)
-        self.use_llm_fallback = kwargs.get('use_llm_fallback', True)
-    
-    def extract_tables(self, html_content: str, base_url: str = "", **kwargs) -> List[Dict[str, Any]]:
-        """Extract tables using smart hybrid approach."""
-        if not html_content:
-            return []
-        
-        soup = BeautifulSoup(html_content, 'lxml')
-        table_tags = soup.find_all('table')
-        
-        if not table_tags:
-            return []
-        
-        extracted_tables = []
-        
-        for idx, table_tag in enumerate(table_tags):
-            try:
-                # Assess table complexity
-                complexity_score = self._assess_table_complexity(table_tag)
-                
-                if complexity_score < self.complexity_threshold:
-                    # Use rule-based extraction for simple tables
-                    table_data = self.default_extractor._extract_table_data(table_tag, base_url, idx)
-                    
-                    if table_data and self.default_extractor._is_valid_table(table_data):
-                        table_data['metadata']['extraction_method'] = 'rule_based'
-                        table_data['metadata']['complexity_score'] = complexity_score
-                        extracted_tables.append(table_data)
-                        
-                        if self.verbose:
-                            logger.debug(f"Table {idx} extracted with rule-based method (complexity: {complexity_score})")
-                    
-                elif self.use_llm_fallback:
-                    # Use LLM for complex tables
-                    table_html = str(table_tag)
-                    table_data = await self.llm_extractor._extract_with_llm(table_html, idx, base_url)
-                    
-                    if table_data:
-                        table_data['metadata']['complexity_score'] = complexity_score
-                        extracted_tables.append(table_data)
-                        
-                        if self.verbose:
-                            logger.debug(f"Table {idx} extracted with LLM method (complexity: {complexity_score})")
-                
-            except Exception as e:
-                logger.error(f"Error extracting table {idx}: {str(e)}")
-                continue
-        
-        logger.info(f"Extracted {len(extracted_tables)} tables using smart extraction")
-        return extracted_tables
-    
-    def _assess_table_complexity(self, table_tag: Tag) -> int:
-        """Assess the complexity of a table to choose extraction method."""
-        complexity = 0
-        
-        # Count nested tables
-        nested_tables = table_tag.find_all('table')
-        complexity += len(nested_tables) * 3
-        
-        # Check for colspan/rowspan
-        cells_with_span = table_tag.find_all(['td', 'th'], attrs={'colspan': True})
-        cells_with_span.extend(table_tag.find_all(['td', 'th'], attrs={'rowspan': True}))
-        complexity += len(cells_with_span) * 2
-        
-        # Count rows and columns
-        rows = table_tag.find_all('tr')
-        if rows:
-            complexity += len(rows) // 5  # Every 5 rows adds complexity
-            
-            max_cols = 0
-            for row in rows:
-                cols = len(row.find_all(['td', 'th']))
-                max_cols = max(max_cols, cols)
-            complexity += max_cols // 3  # Every 3 columns adds complexity
-        
-        # Check for complex content in cells
-        for cell in table_tag.find_all(['td', 'th']):
-            # Lists in cells
-            if cell.find(['ul', 'ol']):
-                complexity += 2
-            
-            # Forms in cells
-            if cell.find(['input', 'select', 'textarea']):
-                complexity += 2
-            
-            # Images in cells
-            if cell.find('img'):
-                complexity += 1
-        
-        # Check for irregular structure
-        row_col_counts = []
-        for row in table_tag.find_all('tr'):
-            col_count = len(row.find_all(['td', 'th']))
-            row_col_counts.append(col_count)
-        
-        if row_col_counts and len(set(row_col_counts)) > 1:
-            complexity += 3  # Irregular column structure
-        
-        return complexity
-
-
-# Factory function for creating table extraction strategies
-def create_table_extraction_strategy(
-    strategy_type: str,
-    config: Optional[Dict[str, Any]] = None
-) -> TableExtractionStrategy:
-    """
-    Factory function to create table extraction strategies.
-    
-    Args:
-        strategy_type: Type of strategy ("default", "llm", "smart", "none")
-        config: Configuration dictionary for the strategy
-        
-    Returns:
-        Configured table extraction strategy instance
-    """
-    config = config or {}
-    
-    strategies = {
-        "default": DefaultTableExtraction,
-        "llm": LLMTableExtraction,
-        "smart": SmartTableExtraction,
-        "none": NoTableExtraction
-    }
-    
-    if strategy_type not in strategies:
-        raise ValueError(f"Unknown table extraction strategy: {strategy_type}. Available: {list(strategies.keys())}")
-    
-    strategy_class = strategies[strategy_type]
-    return strategy_class(**config)
-
-
-# Convenience functions
-async def extract_tables(
-    html_content: str,
-    base_url: str = "",
-    strategy: str = "default",
-    config: Optional[Dict[str, Any]] = None
-) -> List[Dict[str, Any]]:
-    """
-    Extract tables from HTML content using specified strategy.
-    
-    Args:
-        html_content: HTML content to extract tables from
-        base_url: Base URL for resolving relative links
-        strategy: Extraction strategy to use
-        config: Strategy configuration
-        
-    Returns:
-        List of extracted table data
-    """
-    extractor = create_table_extraction_strategy(strategy, config)
-    return extractor.extract_tables(html_content, base_url)
-
-
-def tables_to_markdown(tables: List[Dict[str, Any]]) -> str:
-    """
-    Convert extracted tables to markdown format.
-    
-    Args:
-        tables: List of table data from extraction
-        
-    Returns:
-        Markdown representation of tables
-    """
-    if not tables:
-        return ""
-    
-    markdown_parts = []
-    
-    for i, table in enumerate(tables):
-        # Add table title
-        caption = table.get('caption', f'Table {i + 1}')
-        markdown_parts.append(f"\n## {caption}\n")
-        
-        # Add summary if present
-        summary = table.get('summary')
-        if summary:
-            markdown_parts.append(f"*{summary}*\n")
-        
-        # Create markdown table
-        headers = table.get('headers', [])
-        rows = table.get('rows', [])
-        
-        if headers and rows:
-            # Header row
-            header_row = "| " + " | ".join(str(h) for h in headers) + " |"
-            markdown_parts.append(header_row)
-            
-            # Separator row
-            separator = "| " + " | ".join(["---"] * len(headers)) + " |"
-            markdown_parts.append(separator)
-            
-            # Data rows
-            for row in rows:
-                # Ensure row has same number of columns as headers
-                padded_row = row + [''] * (len(headers) - len(row))
-                padded_row = padded_row[:len(headers)]  # Truncate if too long
-                
-                # Convert cells to strings, handling complex cell data
-                cell_strings = []
-                for cell in padded_row:
-                    if isinstance(cell, dict) and 'text' in cell:
-                        # Handle cells with links
-                        cell_text = cell['text']
-                        if 'links' in cell and cell['links']:
-                            # Add first link as markdown link
-                            first_link = cell['links'][0]
-                            cell_text = f"[{first_link['text']}]({first_link['url']})"
-                        cell_strings.append(cell_text)
-                    else:
-                        cell_strings.append(str(cell))
-                
-                data_row = "| " + " | ".join(cell_strings) + " |"
-                markdown_parts.append(data_row)
-        
-        markdown_parts.append("")  # Empty line between tables
-    
-    return "\n".join(markdown_parts)
diff --git a/apps/backend/app/services/url_seeder.py b/apps/backend/app/services/url_seeder.py
deleted file mode 100644
index ea40b11..0000000
--- a/apps/backend/app/services/url_seeder.py
+++ /dev/null
@@ -1,619 +0,0 @@
-"""
-Advanced URL seeding and discovery system inspired by crawl4ai.
-
-This module provides sophisticated URL discovery capabilities:
-- Sitemap parsing and URL extraction  
-- Common Crawl data integration
-- Pattern-based URL filtering and scoring
-- Concurrent URL validation and scoring
-"""
-
-import asyncio
-import re
-import xml.etree.ElementTree as ET
-from typing import Dict, List, Optional, Set, Any, Tuple
-from urllib.parse import urljoin, urlparse, parse_qs
-from dataclasses import dataclass, field
-import time
-
-import httpx
-import structlog
-from bs4 import BeautifulSoup
-
-from app.utils.text_processing import clean_tokens
-
-logger = structlog.get_logger(__name__)
-
-
-@dataclass
-class SeedingConfig:
-    """Configuration for URL discovery and seeding."""
-    
-    source: str = "sitemap"  # "sitemap", "cc", "sitemap+cc", "crawl"
-    pattern: Optional[str] = None  # URL pattern to match
-    query: Optional[str] = None  # Query for relevance scoring
-    score_threshold: float = 0.0  # Minimum score threshold
-    max_urls: int = 1000  # Maximum URLs to discover
-    concurrent_requests: int = 10  # Concurrent validation requests
-    timeout: float = 10.0  # Request timeout in seconds
-    
-    # Sitemap-specific settings
-    sitemap_urls: List[str] = field(default_factory=list)
-    follow_sitemap_index: bool = True
-    
-    # Common Crawl settings
-    cc_index: Optional[str] = None  # Common Crawl index to use
-    cc_limit: int = 100  # Limit for CC results
-    
-    # Crawl-based discovery settings
-    crawl_depth: int = 2  # Maximum crawl depth
-    crawl_max_pages: int = 50  # Maximum pages to crawl
-    
-    # Filtering settings
-    include_patterns: List[str] = field(default_factory=list)
-    exclude_patterns: List[str] = field(default_factory=list)
-    allowed_domains: List[str] = field(default_factory=list)
-    blocked_domains: List[str] = field(default_factory=list)
-
-
-@dataclass
-class DiscoveredURL:
-    """Represents a discovered URL with metadata."""
-    
-    url: str
-    source: str  # "sitemap", "cc", "crawl"
-    score: float = 0.0
-    last_modified: Optional[str] = None
-    change_freq: Optional[str] = None
-    priority: Optional[float] = None
-    
-    # Additional metadata
-    title: Optional[str] = None
-    description: Optional[str] = None
-    content_type: Optional[str] = None
-    status_code: Optional[int] = None
-    discovery_time: float = field(default_factory=time.time)
-    
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary representation."""
-        return {
-            "url": self.url,
-            "source": self.source, 
-            "score": self.score,
-            "last_modified": self.last_modified,
-            "change_freq": self.change_freq,
-            "priority": self.priority,
-            "title": self.title,
-            "description": self.description,
-            "content_type": self.content_type,
-            "status_code": self.status_code,
-            "discovery_time": self.discovery_time
-        }
-
-
-class URLSeeder:
-    """
-    Advanced URL seeding system for discovering URLs from multiple sources.
-    
-    This class provides comprehensive URL discovery capabilities including
-    sitemap parsing, Common Crawl integration, and pattern-based filtering.
-    """
-    
-    def __init__(self, config: SeedingConfig):
-        """Initialize URL seeder with configuration."""
-        self.config = config
-        self.discovered_urls: Set[str] = set()
-        self.scored_urls: List[DiscoveredURL] = []
-        
-        # HTTP client for requests
-        self.client: Optional[httpx.AsyncClient] = None
-        
-        # Compiled regex patterns for performance
-        self.include_regexes = [re.compile(pattern) for pattern in config.include_patterns]
-        self.exclude_regexes = [re.compile(pattern) for pattern in config.exclude_patterns]
-        
-        # Query tokens for relevance scoring
-        self.query_tokens = set(clean_tokens(config.query.lower().split())) if config.query else set()
-    
-    async def __aenter__(self):
-        """Async context manager entry."""
-        await self.initialize()
-        return self
-    
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit."""
-        await self.close()
-    
-    async def initialize(self):
-        """Initialize HTTP client."""
-        if not self.client:
-            self.client = httpx.AsyncClient(
-                timeout=httpx.Timeout(self.config.timeout),
-                limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
-                headers={
-                    "User-Agent": "Mozilla/5.0 (compatible; URLSeeder/1.0; +https://example.com/bot)"
-                }
-            )
-    
-    async def close(self):
-        """Close HTTP client."""
-        if self.client:
-            await self.client.aclose()
-            self.client = None
-    
-    async def discover(self, base_url: str) -> List[DiscoveredURL]:
-        """
-        Discover URLs from configured sources.
-        
-        Args:
-            base_url: Base URL to start discovery from
-            
-        Returns:
-            List of discovered and scored URLs
-        """
-        logger.info(f"Starting URL discovery for {base_url} with source: {self.config.source}")
-        
-        if not self.client:
-            await self.initialize()
-        
-        discovered_urls = []
-        
-        # Source-based discovery
-        if "sitemap" in self.config.source:
-            sitemap_urls = await self._discover_from_sitemaps(base_url)
-            discovered_urls.extend(sitemap_urls)
-            logger.info(f"Discovered {len(sitemap_urls)} URLs from sitemaps")
-        
-        if "cc" in self.config.source:
-            cc_urls = await self._discover_from_common_crawl(base_url)
-            discovered_urls.extend(cc_urls)
-            logger.info(f"Discovered {len(cc_urls)} URLs from Common Crawl")
-        
-        if self.config.source == "crawl":
-            crawl_urls = await self._discover_from_crawling(base_url)
-            discovered_urls.extend(crawl_urls)
-            logger.info(f"Discovered {len(crawl_urls)} URLs from crawling")
-        
-        # Remove duplicates and apply filtering
-        unique_urls = self._deduplicate_and_filter(discovered_urls)
-        logger.info(f"After deduplication and filtering: {len(unique_urls)} URLs")
-        
-        # Score and rank URLs
-        scored_urls = await self._score_and_rank_urls(unique_urls)
-        
-        # Apply score threshold
-        final_urls = [
-            url for url in scored_urls 
-            if url.score >= self.config.score_threshold
-        ]
-        
-        # Limit results
-        final_urls = final_urls[:self.config.max_urls]
-        
-        logger.info(f"Final URL set: {len(final_urls)} URLs (threshold: {self.config.score_threshold})")
-        
-        return final_urls
-    
-    async def _discover_from_sitemaps(self, base_url: str) -> List[DiscoveredURL]:
-        """Discover URLs from XML sitemaps."""
-        discovered_urls = []
-        
-        # Determine sitemap URLs
-        sitemap_urls = self.config.sitemap_urls.copy()
-        if not sitemap_urls:
-            # Try common sitemap locations
-            common_locations = [
-                "/sitemap.xml",
-                "/sitemap_index.xml",
-                "/sitemaps.xml",
-                "/robots.txt"  # Parse sitemap references
-            ]
-            
-            for location in common_locations:
-                sitemap_url = urljoin(base_url, location)
-                if location.endswith("robots.txt"):
-                    # Extract sitemap URLs from robots.txt
-                    robots_sitemaps = await self._extract_sitemaps_from_robots(sitemap_url)
-                    sitemap_urls.extend(robots_sitemaps)
-                else:
-                    sitemap_urls.append(sitemap_url)
-        
-        # Process each sitemap URL
-        for sitemap_url in sitemap_urls:
-            try:
-                urls_from_sitemap = await self._parse_sitemap(sitemap_url)
-                discovered_urls.extend(urls_from_sitemap)
-                
-                if len(discovered_urls) >= self.config.max_urls:
-                    break
-                    
-            except Exception as e:
-                logger.warning(f"Failed to parse sitemap {sitemap_url}: {str(e)}")
-                continue
-        
-        return discovered_urls
-    
-    async def _parse_sitemap(self, sitemap_url: str) -> List[DiscoveredURL]:
-        """Parse a single XML sitemap."""
-        try:
-            response = await self.client.get(sitemap_url)
-            response.raise_for_status()
-            
-            content = response.text
-            root = ET.fromstring(content)
-            
-            # Handle namespace
-            namespace = ""
-            if root.tag.startswith("{"):
-                namespace = root.tag.split("}")[0] + "}"
-            
-            discovered_urls = []
-            
-            # Check if this is a sitemap index
-            sitemap_elements = root.findall(f"{namespace}sitemap")
-            if sitemap_elements and self.config.follow_sitemap_index:
-                # This is a sitemap index - recursively parse child sitemaps
-                for sitemap_elem in sitemap_elements:
-                    loc_elem = sitemap_elem.find(f"{namespace}loc")
-                    if loc_elem is not None:
-                        child_sitemap_url = loc_elem.text.strip()
-                        child_urls = await self._parse_sitemap(child_sitemap_url)
-                        discovered_urls.extend(child_urls)
-            else:
-                # This is a regular sitemap - extract URLs
-                url_elements = root.findall(f"{namespace}url")
-                for url_elem in url_elements:
-                    loc_elem = url_elem.find(f"{namespace}loc")
-                    if loc_elem is not None:
-                        url = loc_elem.text.strip()
-                        
-                        # Extract additional metadata
-                        lastmod_elem = url_elem.find(f"{namespace}lastmod")
-                        changefreq_elem = url_elem.find(f"{namespace}changefreq") 
-                        priority_elem = url_elem.find(f"{namespace}priority")
-                        
-                        discovered_url = DiscoveredURL(
-                            url=url,
-                            source="sitemap",
-                            last_modified=lastmod_elem.text.strip() if lastmod_elem is not None else None,
-                            change_freq=changefreq_elem.text.strip() if changefreq_elem is not None else None,
-                            priority=float(priority_elem.text.strip()) if priority_elem is not None else None
-                        )
-                        
-                        discovered_urls.append(discovered_url)
-            
-            return discovered_urls
-            
-        except Exception as e:
-            logger.error(f"Error parsing sitemap {sitemap_url}: {str(e)}")
-            return []
-    
-    async def _extract_sitemaps_from_robots(self, robots_url: str) -> List[str]:
-        """Extract sitemap URLs from robots.txt."""
-        try:
-            response = await self.client.get(robots_url)
-            if response.status_code != 200:
-                return []
-            
-            sitemaps = []
-            for line in response.text.split('\n'):
-                line = line.strip()
-                if line.lower().startswith('sitemap:'):
-                    sitemap_url = line[8:].strip()
-                    sitemaps.append(sitemap_url)
-            
-            return sitemaps
-            
-        except Exception as e:
-            logger.warning(f"Error parsing robots.txt {robots_url}: {str(e)}")
-            return []
-    
-    async def _discover_from_common_crawl(self, base_url: str) -> List[DiscoveredURL]:
-        """Discover URLs from Common Crawl data."""
-        # This is a simplified implementation
-        # In production, you would integrate with Common Crawl's APIs
-        logger.info("Common Crawl integration not fully implemented - using mock data")
-        
-        # Mock implementation
-        await asyncio.sleep(0.1)  # Simulate API call
-        
-        domain = urlparse(base_url).netloc
-        mock_urls = [
-            DiscoveredURL(f"https://{domain}/page1", "cc", score=0.8),
-            DiscoveredURL(f"https://{domain}/page2", "cc", score=0.6),
-            DiscoveredURL(f"https://{domain}/blog/", "cc", score=0.7),
-        ]
-        
-        return mock_urls
-    
-    async def _discover_from_crawling(self, base_url: str) -> List[DiscoveredURL]:
-        """Discover URLs by crawling the website."""
-        discovered_urls = []
-        crawled_urls = set()
-        urls_to_crawl = [base_url]
-        current_depth = 0
-        
-        while urls_to_crawl and current_depth < self.config.crawl_depth:
-            current_level_urls = urls_to_crawl.copy()
-            urls_to_crawl.clear()
-            
-            # Process current level
-            for url in current_level_urls:
-                if url in crawled_urls or len(crawled_urls) >= self.config.crawl_max_pages:
-                    continue
-                
-                try:
-                    response = await self.client.get(url)
-                    if response.status_code != 200:
-                        continue
-                    
-                    crawled_urls.add(url)
-                    
-                    # Add current URL to discovered
-                    discovered_urls.append(DiscoveredURL(
-                        url=url,
-                        source="crawl",
-                        status_code=response.status_code,
-                        content_type=response.headers.get("content-type", "")
-                    ))
-                    
-                    # Extract links for next level
-                    if current_depth < self.config.crawl_depth - 1:
-                        soup = BeautifulSoup(response.text, 'lxml')
-                        for link in soup.find_all('a', href=True):
-                            href = link['href']
-                            absolute_url = urljoin(url, href)
-                            
-                            # Only follow same-domain links
-                            if urlparse(absolute_url).netloc == urlparse(base_url).netloc:
-                                if absolute_url not in crawled_urls:
-                                    urls_to_crawl.append(absolute_url)
-                
-                except Exception as e:
-                    logger.warning(f"Error crawling {url}: {str(e)}")
-                    continue
-            
-            current_depth += 1
-        
-        return discovered_urls
-    
-    def _deduplicate_and_filter(self, discovered_urls: List[DiscoveredURL]) -> List[DiscoveredURL]:
-        """Remove duplicates and apply filtering rules."""
-        unique_urls = {}
-        
-        for discovered_url in discovered_urls:
-            url = discovered_url.url
-            
-            # Skip if already seen
-            if url in unique_urls:
-                continue
-            
-            # Apply pattern filtering
-            if not self._passes_pattern_filters(url):
-                continue
-            
-            # Apply domain filtering  
-            if not self._passes_domain_filters(url):
-                continue
-            
-            unique_urls[url] = discovered_url
-        
-        return list(unique_urls.values())
-    
-    def _passes_pattern_filters(self, url: str) -> bool:
-        """Check if URL passes pattern-based filters."""
-        # Check include patterns
-        if self.include_regexes:
-            if not any(regex.search(url) for regex in self.include_regexes):
-                return False
-        
-        # Check exclude patterns
-        if self.exclude_regexes:
-            if any(regex.search(url) for regex in self.exclude_regexes):
-                return False
-        
-        # Check explicit pattern from config
-        if self.config.pattern:
-            pattern_regex = re.compile(self.config.pattern)
-            if not pattern_regex.search(url):
-                return False
-        
-        return True
-    
-    def _passes_domain_filters(self, url: str) -> bool:
-        """Check if URL passes domain-based filters."""
-        domain = urlparse(url).netloc.lower()
-        
-        # Check allowed domains
-        if self.config.allowed_domains:
-            if not any(allowed in domain for allowed in self.config.allowed_domains):
-                return False
-        
-        # Check blocked domains
-        if self.config.blocked_domains:
-            if any(blocked in domain for blocked in self.config.blocked_domains):
-                return False
-        
-        return True
-    
-    async def _score_and_rank_urls(self, urls: List[DiscoveredURL]) -> List[DiscoveredURL]:
-        """Score and rank URLs based on various factors."""
-        if not urls:
-            return []
-        
-        # Score URLs concurrently for performance
-        semaphore = asyncio.Semaphore(self.config.concurrent_requests)
-        
-        async def score_url(discovered_url: DiscoveredURL) -> DiscoveredURL:
-            async with semaphore:
-                try:
-                    score = await self._calculate_url_score(discovered_url)
-                    discovered_url.score = score
-                    return discovered_url
-                except Exception as e:
-                    logger.warning(f"Error scoring URL {discovered_url.url}: {str(e)}")
-                    discovered_url.score = 0.0
-                    return discovered_url
-        
-        # Score all URLs
-        scored_urls = await asyncio.gather(*[score_url(url) for url in urls])
-        
-        # Sort by score (descending)
-        scored_urls.sort(key=lambda x: x.score, reverse=True)
-        
-        return scored_urls
-    
-    async def _calculate_url_score(self, discovered_url: DiscoveredURL) -> float:
-        """Calculate relevance score for a URL."""
-        score = 0.0
-        url = discovered_url.url
-        
-        # Base score from source
-        source_scores = {
-            "sitemap": 0.8,
-            "cc": 0.6,
-            "crawl": 0.5
-        }
-        score += source_scores.get(discovered_url.source, 0.5)
-        
-        # Sitemap-specific scoring
-        if discovered_url.source == "sitemap":
-            if discovered_url.priority:
-                score += discovered_url.priority * 0.2
-            
-            if discovered_url.change_freq:
-                freq_scores = {
-                    "always": 0.1, "hourly": 0.09, "daily": 0.08,
-                    "weekly": 0.06, "monthly": 0.04, "yearly": 0.02, "never": 0.0
-                }
-                score += freq_scores.get(discovered_url.change_freq.lower(), 0.0)
-        
-        # URL structure scoring
-        score += self._score_url_structure(url)
-        
-        # Query relevance scoring
-        if self.query_tokens:
-            score += self._score_query_relevance(url)
-        
-        # Content preview scoring (if enabled)
-        if self.config.source == "crawl":
-            content_score = await self._score_content_preview(discovered_url)
-            score += content_score
-        
-        return min(1.0, score)  # Cap at 1.0
-    
-    def _score_url_structure(self, url: str) -> float:
-        """Score URL based on structure indicators."""
-        score = 0.0
-        path = urlparse(url).path.lower()
-        
-        # Content indicators
-        content_indicators = [
-            "blog", "article", "post", "news", "guide", "tutorial",
-            "documentation", "docs", "help", "support", "about"
-        ]
-        
-        for indicator in content_indicators:
-            if indicator in path:
-                score += 0.1
-                break
-        
-        # Depth penalty (deeper URLs less likely to be important)
-        depth = len([p for p in path.split('/') if p])
-        if depth <= 2:
-            score += 0.1
-        elif depth > 4:
-            score -= 0.1
-        
-        # File extension analysis
-        if path.endswith(('.html', '.htm', '.php')):
-            score += 0.05
-        elif path.endswith(('.pdf', '.doc', '.docx')):
-            score += 0.03
-        
-        return score
-    
-    def _score_query_relevance(self, url: str) -> float:
-        """Score URL relevance to the query."""
-        if not self.query_tokens:
-            return 0.0
-        
-        # Extract words from URL
-        url_words = set(re.findall(r'[a-zA-Z]+', url.lower()))
-        
-        # Calculate overlap with query
-        overlap = len(self.query_tokens.intersection(url_words))
-        max_overlap = len(self.query_tokens)
-        
-        if max_overlap > 0:
-            return (overlap / max_overlap) * 0.3
-        
-        return 0.0
-    
-    async def _score_content_preview(self, discovered_url: DiscoveredURL) -> float:
-        """Score URL based on content preview."""
-        # This would make a HEAD request to get basic content info
-        # For now, return a simple score based on status code
-        if discovered_url.status_code == 200:
-            return 0.1
-        elif discovered_url.status_code in [301, 302, 307, 308]:
-            return 0.05
-        else:
-            return 0.0
-
-
-# Factory function
-async def discover_urls(
-    base_url: str,
-    source: str = "sitemap",
-    pattern: Optional[str] = None,
-    query: Optional[str] = None,
-    max_urls: int = 100
-) -> List[DiscoveredURL]:
-    """
-    Convenience function to discover URLs.
-    
-    Args:
-        base_url: Base URL to start discovery
-        source: Discovery source ("sitemap", "cc", "crawl")
-        pattern: URL pattern to match
-        query: Query for relevance scoring
-        max_urls: Maximum URLs to return
-        
-    Returns:
-        List of discovered URLs
-    """
-    config = SeedingConfig(
-        source=source,
-        pattern=pattern,
-        query=query,
-        max_urls=max_urls
-    )
-    
-    async with URLSeeder(config) as seeder:
-        return await seeder.discover(base_url)
-
-
-# Utility functions
-def filter_urls_by_patterns(
-    urls: List[str],
-    include_patterns: List[str] = None,
-    exclude_patterns: List[str] = None
-) -> List[str]:
-    """Filter URLs by regex patterns."""
-    include_regexes = [re.compile(p) for p in (include_patterns or [])]
-    exclude_regexes = [re.compile(p) for p in (exclude_patterns or [])]
-    
-    filtered = []
-    for url in urls:
-        # Check include patterns
-        if include_regexes and not any(r.search(url) for r in include_regexes):
-            continue
-        
-        # Check exclude patterns
-        if exclude_regexes and any(r.search(url) for r in exclude_regexes):
-            continue
-        
-        filtered.append(url)
-    
-    return filtered
diff --git a/apps/backend/app/services/user_agent_generator.py b/apps/backend/app/services/user_agent_generator.py
deleted file mode 100644
index 289c0e3..0000000
--- a/apps/backend/app/services/user_agent_generator.py
+++ /dev/null
@@ -1,639 +0,0 @@
-"""
-Advanced user agent generation system with multiple strategies and client hints.
-
-This module provides sophisticated user agent management:
-- Multiple generation strategies (Valid, Online, Custom)
-- Automatic client hints generation (Sec-CH-UA)
-- Browser fingerprinting avoidance
-- Platform and browser specific agents
-- Performance optimization with caching
-"""
-
-import random
-import re
-import json
-from abc import ABC, abstractmethod
-from typing import Optional, List, Dict, Union, Tuple
-from dataclasses import dataclass
-from datetime import datetime, timedelta
-from pathlib import Path
-
-import structlog
-
-logger = structlog.get_logger(__name__)
-
-
-@dataclass
-class UserAgentProfile:
-    """Profile for user agent generation."""
-    browser: str
-    version: str
-    platform: str
-    os: str
-    engine: str
-    full_agent: str
-    client_hints: str
-    popularity_score: float = 0.0
-    
-    def to_dict(self) -> Dict[str, str]:
-        """Convert to dictionary format."""
-        return {
-            'browser': self.browser,
-            'version': self.version,
-            'platform': self.platform,
-            'os': self.os,
-            'engine': self.engine,
-            'user_agent': self.full_agent,
-            'client_hints': self.client_hints
-        }
-
-
-class UAGenerator(ABC):
-    """Abstract base class for user agent generators."""
-    
-    @abstractmethod
-    def generate(self, 
-                browsers: Optional[List[str]] = None,
-                os: Optional[Union[str, List[str]]] = None,
-                min_version: float = 0.0,
-                platforms: Optional[Union[str, List[str]]] = None,
-                pct_threshold: Optional[float] = None,
-                fallback: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36") -> Union[str, UserAgentProfile]:
-        """Generate user agent string or profile."""
-        pass
-    
-    @staticmethod
-    def generate_client_hints(user_agent: str) -> str:
-        """Generate Sec-CH-UA header value based on user agent string."""
-        def _parse_user_agent(user_agent: str) -> Dict[str, str]:
-            """Parse a user agent string to extract browser and version information."""
-            browsers = {
-                "chrome": r"Chrome/(\d+)",
-                "edge": r"Edg/(\d+)", 
-                "safari": r"Version/(\d+)",
-                "firefox": r"Firefox/(\d+)",
-            }
-            
-            result = {}
-            for browser, pattern in browsers.items():
-                match = re.search(pattern, user_agent)
-                if match:
-                    result[browser] = match.group(1)
-            
-            return result
-        
-        browsers = _parse_user_agent(user_agent)
-        
-        # Client hints components
-        hints = []
-        
-        # Handle different browser combinations
-        if "chrome" in browsers:
-            hints.append(f'"Chromium";v="{browsers["chrome"]}"')
-            hints.append('"Not_A Brand";v="8"')
-            
-            if "edge" in browsers:
-                hints.append(f'"Microsoft Edge";v="{browsers["edge"]}"')
-            else:
-                hints.append(f'"Google Chrome";v="{browsers["chrome"]}"')
-        
-        elif "firefox" in browsers:
-            # Firefox doesn't typically send Sec-CH-UA
-            return '""'
-        
-        elif "safari" in browsers:
-            # Safari's format for client hints
-            hints.append(f'"Safari";v="{browsers["safari"]}"')
-            hints.append('"Not_A Brand";v="8"')
-        
-        return ", ".join(hints)
-    
-    @staticmethod
-    def parse_user_agent_details(user_agent: str) -> UserAgentProfile:
-        """Parse user agent string into detailed profile."""
-        # Default values
-        browser = "Unknown"
-        version = "0.0"
-        platform = "Unknown"
-        os = "Unknown"
-        engine = "Unknown"
-        
-        # Browser detection patterns
-        browser_patterns = {
-            'Chrome': r'Chrome/(\d+\.\d+)',
-            'Firefox': r'Firefox/(\d+\.\d+)',
-            'Safari': r'Version/(\d+\.\d+).*Safari',
-            'Edge': r'Edg/(\d+\.\d+)',
-            'Opera': r'OPR/(\d+\.\d+)',
-            'Internet Explorer': r'MSIE (\d+\.\d+)',
-        }
-        
-        # OS detection patterns
-        os_patterns = {
-            'Windows': r'Windows NT (\d+\.\d+)',
-            'Mac OS': r'Mac OS X (\d+[_\d]*)',
-            'Linux': r'Linux',
-            'Android': r'Android (\d+\.\d+)',
-            'iOS': r'iPhone OS (\d+[_\d]*)',
-        }
-        
-        # Platform detection patterns
-        platform_patterns = {
-            'Desktop': r'(Windows|Mac OS X|Linux)',
-            'Mobile': r'(Android|iPhone|iPad)',
-            'Tablet': r'(iPad|Android.*Tablet)',
-        }
-        
-        # Engine detection patterns
-        engine_patterns = {
-            'WebKit': r'WebKit/(\d+\.\d+)',
-            'Gecko': r'Gecko/(\d+)',
-            'Blink': r'Chrome.*WebKit',  # Chrome uses Blink (fork of WebKit)
-            'EdgeHTML': r'Edge/(\d+\.\d+)',
-        }
-        
-        # Detect browser
-        for browser_name, pattern in browser_patterns.items():
-            match = re.search(pattern, user_agent)
-            if match:
-                browser = browser_name
-                version = match.group(1)
-                break
-        
-        # Detect OS
-        for os_name, pattern in os_patterns.items():
-            if re.search(pattern, user_agent):
-                os = os_name
-                break
-        
-        # Detect platform
-        for platform_name, pattern in platform_patterns.items():
-            if re.search(pattern, user_agent):
-                platform = platform_name
-                break
-        
-        # Detect engine
-        for engine_name, pattern in engine_patterns.items():
-            if re.search(pattern, user_agent):
-                engine = engine_name
-                break
-        
-        # Generate client hints
-        client_hints = UAGenerator.generate_client_hints(user_agent)
-        
-        return UserAgentProfile(
-            browser=browser,
-            version=version,
-            platform=platform,
-            os=os,
-            engine=engine,
-            full_agent=user_agent,
-            client_hints=client_hints
-        )
-
-
-class ValidUAGenerator(UAGenerator):
-    """User agent generator using fake-useragent library with validation."""
-    
-    def __init__(self):
-        """Initialize with fallback agents if fake-useragent is not available."""
-        self._fallback_agents = [
-            # Chrome agents
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-            
-            # Firefox agents
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/121.0",
-            "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/121.0",
-            
-            # Edge agents
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
-            
-            # Safari agents
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
-        ]
-        
-        # Try to import fake-useragent
-        try:
-            from fake_useragent import UserAgent
-            self.ua = UserAgent()
-            self._has_fake_ua = True
-        except ImportError:
-            logger.warning("fake-useragent not available, using fallback agents")
-            self.ua = None
-            self._has_fake_ua = False
-    
-    def generate(self,
-                browsers: Optional[List[str]] = None,
-                os: Optional[Union[str, List[str]]] = None,
-                min_version: float = 0.0,
-                platforms: Optional[Union[str, List[str]]] = None,
-                pct_threshold: Optional[float] = None,
-                fallback: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36") -> str:
-        """Generate user agent string using fake-useragent or fallbacks."""
-        
-        if self._has_fake_ua and self.ua:
-            try:
-                # Use fake-useragent with specified parameters
-                if browsers:
-                    # Try to get specific browser
-                    browser_name = random.choice(browsers).lower()
-                    if hasattr(self.ua, browser_name):
-                        return getattr(self.ua, browser_name)
-                
-                # Get random user agent
-                return self.ua.random
-                
-            except Exception as e:
-                logger.warning(f"fake-useragent failed: {str(e)}, using fallback")
-        
-        # Filter fallback agents by criteria
-        suitable_agents = self._fallback_agents
-        
-        if browsers:
-            browser_filters = [b.lower() for b in browsers]
-            suitable_agents = [
-                agent for agent in suitable_agents
-                if any(browser in agent.lower() for browser in browser_filters)
-            ]
-        
-        if os:
-            os_list = [os] if isinstance(os, str) else os
-            os_filters = [o.lower() for o in os_list]
-            suitable_agents = [
-                agent for agent in suitable_agents
-                if any(os_name in agent.lower() for os_name in os_filters)
-            ]
-        
-        # Return random suitable agent or fallback
-        if suitable_agents:
-            return random.choice(suitable_agents)
-        else:
-            return fallback
-
-
-class OnlineUAGenerator(UAGenerator):
-    """User agent generator that fetches fresh agents from online sources."""
-    
-    def __init__(self, cache_file: Optional[str] = None, cache_duration: int = 24):
-        """
-        Initialize online UA generator.
-        
-        Args:
-            cache_file: Path to cache file for offline usage
-            cache_duration: Cache duration in hours
-        """
-        self.cache_file = cache_file
-        self.cache_duration = cache_duration
-        self.agents: List[str] = []
-        self.last_fetch: Optional[datetime] = None
-        
-        # Load cached agents if available
-        self._load_cache()
-    
-    def _load_cache(self):
-        """Load cached user agents from file."""
-        if not self.cache_file or not Path(self.cache_file).exists():
-            return
-        
-        try:
-            with open(self.cache_file, 'r', encoding='utf-8') as f:
-                data = json.load(f)
-                self.agents = data.get('agents', [])
-                last_fetch_str = data.get('last_fetch')
-                if last_fetch_str:
-                    self.last_fetch = datetime.fromisoformat(last_fetch_str)
-        except Exception as e:
-            logger.warning(f"Failed to load cached user agents: {str(e)}")
-    
-    def _save_cache(self):
-        """Save user agents to cache file."""
-        if not self.cache_file:
-            return
-        
-        try:
-            Path(self.cache_file).parent.mkdir(parents=True, exist_ok=True)
-            
-            data = {
-                'agents': self.agents,
-                'last_fetch': self.last_fetch.isoformat() if self.last_fetch else None
-            }
-            
-            with open(self.cache_file, 'w', encoding='utf-8') as f:
-                json.dump(data, f, indent=2)
-        except Exception as e:
-            logger.warning(f"Failed to save user agents cache: {str(e)}")
-    
-    def _needs_refresh(self) -> bool:
-        """Check if cache needs refresh."""
-        if not self.agents or not self.last_fetch:
-            return True
-        
-        age = datetime.now() - self.last_fetch
-        return age > timedelta(hours=self.cache_duration)
-    
-    def _fetch_agents(self):
-        """Fetch user agents from online sources."""
-        try:
-            import requests
-            from bs4 import BeautifulSoup
-        except ImportError:
-            logger.error("requests and beautifulsoup4 required for online UA generation")
-            return
-        
-        # Try multiple sources
-        sources = [
-            {
-                'url': 'https://www.useragents.me/',
-                'selector': '.ua'
-            },
-            {
-                'url': 'https://developers.whatismybrowser.com/useragents/explore/',
-                'selector': '.useragent'
-            }
-        ]
-        
-        new_agents = []
-        
-        for source in sources:
-            try:
-                response = requests.get(
-                    source['url'],
-                    timeout=10,
-                    headers={'Accept': 'text/html,application/xhtml+xml'}
-                )
-                response.raise_for_status()
-                
-                soup = BeautifulSoup(response.content, 'html.parser')
-                elements = soup.select(source['selector'])
-                
-                for element in elements[:50]:  # Limit to 50 per source
-                    agent_text = element.get_text().strip()
-                    if agent_text and len(agent_text) > 50:  # Basic validation
-                        new_agents.append(agent_text)
-                
-                if new_agents:
-                    logger.info(f"Fetched {len(new_agents)} user agents from {source['url']}")
-                    break  # Use first successful source
-                    
-            except Exception as e:
-                logger.warning(f"Failed to fetch from {source['url']}: {str(e)}")
-                continue
-        
-        if new_agents:
-            self.agents = new_agents
-            self.last_fetch = datetime.now()
-            self._save_cache()
-        else:
-            logger.warning("Failed to fetch user agents from all sources")
-    
-    def generate(self,
-                browsers: Optional[List[str]] = None,
-                os: Optional[Union[str, List[str]]] = None,
-                min_version: float = 0.0,
-                platforms: Optional[Union[str, List[str]]] = None,
-                pct_threshold: Optional[float] = None,
-                fallback: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36") -> str:
-        """Generate user agent from online sources."""
-        
-        # Refresh cache if needed
-        if self._needs_refresh():
-            self._fetch_agents()
-        
-        if not self.agents:
-            return fallback
-        
-        # Filter agents by criteria
-        suitable_agents = self.agents
-        
-        if browsers:
-            browser_filters = [b.lower() for b in browsers]
-            suitable_agents = [
-                agent for agent in suitable_agents
-                if any(browser in agent.lower() for browser in browser_filters)
-            ]
-        
-        if os:
-            os_list = [os] if isinstance(os, str) else os
-            os_filters = [o.lower() for o in os_list]
-            suitable_agents = [
-                agent for agent in suitable_agents
-                if any(os_name in agent.lower() for os_name in os_filters)
-            ]
-        
-        # Return random suitable agent
-        if suitable_agents:
-            return random.choice(suitable_agents)
-        else:
-            return random.choice(self.agents) if self.agents else fallback
-
-
-class CustomUAGenerator(UAGenerator):
-    """Custom user agent generator with predefined patterns."""
-    
-    def __init__(self, custom_agents: List[str] = None):
-        """
-        Initialize with custom agent list.
-        
-        Args:
-            custom_agents: List of custom user agent strings
-        """
-        self.custom_agents = custom_agents or []
-        
-        # Add some high-quality default agents if none provided
-        if not self.custom_agents:
-            self.custom_agents = [
-                # Latest Chrome versions
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-                
-                # Latest Firefox versions
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
-                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/121.0",
-                
-                # Mobile agents
-                "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1",
-                "Mozilla/5.0 (Linux; Android 13; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
-            ]
-    
-    def add_agent(self, agent: str):
-        """Add custom user agent."""
-        if agent not in self.custom_agents:
-            self.custom_agents.append(agent)
-    
-    def remove_agent(self, agent: str):
-        """Remove custom user agent."""
-        if agent in self.custom_agents:
-            self.custom_agents.remove(agent)
-    
-    def generate(self,
-                browsers: Optional[List[str]] = None,
-                os: Optional[Union[str, List[str]]] = None,
-                min_version: float = 0.0,
-                platforms: Optional[Union[str, List[str]]] = None,
-                pct_threshold: Optional[float] = None,
-                fallback: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36") -> str:
-        """Generate user agent from custom list."""
-        
-        if not self.custom_agents:
-            return fallback
-        
-        # Filter agents by criteria
-        suitable_agents = self.custom_agents
-        
-        if browsers:
-            browser_filters = [b.lower() for b in browsers]
-            suitable_agents = [
-                agent for agent in suitable_agents
-                if any(browser in agent.lower() for browser in browser_filters)
-            ]
-        
-        if os:
-            os_list = [os] if isinstance(os, str) else os
-            os_filters = [o.lower() for o in os_list]
-            suitable_agents = [
-                agent for agent in suitable_agents
-                if any(os_name in agent.lower() for os_name in os_filters)
-            ]
-        
-        # Return random suitable agent
-        return random.choice(suitable_agents) if suitable_agents else fallback
-
-
-class UserAgentManager:
-    """
-    Comprehensive user agent management system.
-    
-    Provides multiple generation strategies and advanced features.
-    """
-    
-    def __init__(self, 
-                 default_strategy: str = "valid",
-                 cache_dir: Optional[str] = None):
-        """
-        Initialize user agent manager.
-        
-        Args:
-            default_strategy: Default generation strategy
-            cache_dir: Directory for caching online agents
-        """
-        self.default_strategy = default_strategy
-        self.cache_dir = cache_dir
-        
-        # Initialize generators
-        self.generators = {
-            'valid': ValidUAGenerator(),
-            'online': OnlineUAGenerator(
-                cache_file=f"{cache_dir}/online_agents.json" if cache_dir else None
-            ),
-            'custom': CustomUAGenerator()
-        }
-        
-        # Usage statistics
-        self.usage_stats = {strategy: 0 for strategy in self.generators.keys()}
-    
-    def generate(self, 
-                strategy: Optional[str] = None,
-                return_profile: bool = False,
-                **kwargs) -> Union[str, UserAgentProfile]:
-        """
-        Generate user agent using specified strategy.
-        
-        Args:
-            strategy: Generation strategy to use
-            return_profile: Whether to return detailed profile
-            **kwargs: Additional parameters for generation
-            
-        Returns:
-            User agent string or profile
-        """
-        strategy = strategy or self.default_strategy
-        
-        if strategy not in self.generators:
-            logger.warning(f"Unknown strategy '{strategy}', using default")
-            strategy = self.default_strategy
-        
-        # Generate user agent
-        generator = self.generators[strategy]
-        user_agent = generator.generate(**kwargs)
-        
-        # Update usage statistics
-        self.usage_stats[strategy] += 1
-        
-        # Return profile if requested
-        if return_profile:
-            return UAGenerator.parse_user_agent_details(user_agent)
-        else:
-            return user_agent
-    
-    def add_custom_agent(self, agent: str):
-        """Add custom user agent to custom generator."""
-        custom_gen = self.generators['custom']
-        if isinstance(custom_gen, CustomUAGenerator):
-            custom_gen.add_agent(agent)
-    
-    def get_random_profile(self, **kwargs) -> UserAgentProfile:
-        """Get random user agent profile with all details."""
-        return self.generate(return_profile=True, **kwargs)
-    
-    def get_mobile_agent(self, **kwargs) -> str:
-        """Get mobile-specific user agent."""
-        kwargs['platforms'] = ['mobile']
-        return self.generate(**kwargs)
-    
-    def get_desktop_agent(self, **kwargs) -> str:
-        """Get desktop-specific user agent."""
-        kwargs['platforms'] = ['desktop']
-        return self.generate(**kwargs)
-    
-    def get_chrome_agent(self, **kwargs) -> str:
-        """Get Chrome-specific user agent."""
-        kwargs['browsers'] = ['Chrome']
-        return self.generate(**kwargs)
-    
-    def get_firefox_agent(self, **kwargs) -> str:
-        """Get Firefox-specific user agent."""
-        kwargs['browsers'] = ['Firefox']
-        return self.generate(**kwargs)
-    
-    def get_usage_stats(self) -> Dict[str, int]:
-        """Get usage statistics for different strategies."""
-        return self.usage_stats.copy()
-    
-    def reset_stats(self):
-        """Reset usage statistics."""
-        self.usage_stats = {strategy: 0 for strategy in self.generators.keys()}
-
-
-# Singleton instance
-_ua_manager: Optional[UserAgentManager] = None
-
-
-def get_user_agent_manager(**kwargs) -> UserAgentManager:
-    """Get singleton user agent manager."""
-    global _ua_manager
-    if _ua_manager is None:
-        _ua_manager = UserAgentManager(**kwargs)
-    return _ua_manager
-
-
-# Convenience functions
-def generate_user_agent(strategy: str = "valid", **kwargs) -> str:
-    """Generate user agent using specified strategy."""
-    manager = get_user_agent_manager()
-    return manager.generate(strategy=strategy, **kwargs)
-
-
-def get_random_user_agent(**kwargs) -> str:
-    """Get random user agent with optional filtering."""
-    manager = get_user_agent_manager()
-    return manager.generate(**kwargs)
-
-
-def get_user_agent_with_hints(strategy: str = "valid", **kwargs) -> Tuple[str, str]:
-    """Get user agent string with client hints."""
-    manager = get_user_agent_manager()
-    profile = manager.generate(strategy=strategy, return_profile=True, **kwargs)
-    return profile.full_agent, profile.client_hints
diff --git a/apps/backend/app/services/virtual_scrolling.py b/apps/backend/app/services/virtual_scrolling.py
deleted file mode 100644
index b13400e..0000000
--- a/apps/backend/app/services/virtual_scrolling.py
+++ /dev/null
@@ -1,627 +0,0 @@
-"""
-Virtual scrolling support for infinite pages inspired by crawl4ai.
-
-This module implements sophisticated virtual scrolling capabilities:
-- Automatic infinite scroll detection and handling
-- Smart waiting strategies for dynamic content
-- Content extraction during scrolling
-- Progress tracking and optimization
-"""
-
-import asyncio
-import json
-import time
-from dataclasses import dataclass
-from typing import Dict, List, Optional, Any, Callable, Tuple
-from urllib.parse import urlparse
-
-import structlog
-from bs4 import BeautifulSoup
-
-logger = structlog.get_logger(__name__)
-
-
-@dataclass
-class VirtualScrollConfig:
-    """Configuration for virtual scrolling behavior."""
-    container_selector: Optional[str] = None
-    scroll_count: int = 10
-    scroll_by: str = "viewport_height"  # "viewport_height", "container_height", "pixels"
-    scroll_pixels: int = 1000
-    wait_after_scroll: float = 2.0
-    wait_for_selector: Optional[str] = None
-    scroll_timeout: float = 30.0
-    check_content_changes: bool = True
-    min_content_increase: int = 100  # Minimum chars to consider new content
-    max_scroll_attempts: int = 50
-    auto_detect_infinite_scroll: bool = True
-    scroll_pause_detection: bool = True
-    content_stabilization_time: float = 3.0
-
-
-@dataclass
-class ScrollState:
-    """State tracking for virtual scrolling operations."""
-    current_scroll: int = 0
-    total_content_length: int = 0
-    last_content_change: float = 0
-    scroll_history: List[int] = None
-    content_snapshots: List[str] = None
-    stabilization_count: int = 0
-    last_successful_scroll: int = 0
-    is_infinite_scroll_detected: bool = False
-    scroll_triggers: List[str] = None
-    
-    def __post_init__(self):
-        if self.scroll_history is None:
-            self.scroll_history = []
-        if self.content_snapshots is None:
-            self.content_snapshots = []
-        if self.scroll_triggers is None:
-            self.scroll_triggers = []
-
-
-@dataclass
-class VirtualScrollResult:
-    """Result of virtual scrolling operation."""
-    success: bool
-    total_scrolls: int
-    final_content: str
-    content_blocks: List[Dict[str, Any]]
-    scroll_metadata: Dict[str, Any]
-    error_message: Optional[str] = None
-    performance_metrics: Optional[Dict[str, Any]] = None
-
-
-class VirtualScrollHandler:
-    """
-    Handler for virtual scrolling operations.
-    
-    This class manages the complex process of scrolling through infinite
-    pages and extracting content as it loads dynamically.
-    """
-    
-    def __init__(self, config: VirtualScrollConfig):
-        """Initialize virtual scroll handler."""
-        self.config = config
-        
-        # JavaScript snippets for scrolling operations
-        self.scroll_js = {
-            "viewport_height": """
-                window.scrollBy(0, window.innerHeight);
-                return window.pageYOffset;
-            """,
-            "container_height": """
-                const container = document.querySelector(arguments[0]);
-                if (container) {
-                    container.scrollBy(0, container.clientHeight);
-                    return container.scrollTop;
-                }
-                return -1;
-            """,
-            "pixels": """
-                window.scrollBy(0, arguments[0]);
-                return window.pageYOffset;
-            """,
-            "to_bottom": """
-                window.scrollTo(0, document.body.scrollHeight);
-                return window.pageYOffset;
-            """,
-            "get_scroll_position": """
-                return {
-                    scrollY: window.pageYOffset,
-                    scrollHeight: document.body.scrollHeight,
-                    clientHeight: window.innerHeight
-                };
-            """,
-            "detect_infinite_scroll": """
-                // Look for common infinite scroll indicators
-                const indicators = [
-                    '[data-testid*="feed"]',
-                    '.infinite-scroll',
-                    '[class*="infinite"]',
-                    '[id*="infinite"]',
-                    '.lazy-load',
-                    '[data-lazy]',
-                    '.loading-more',
-                    '.load-more'
-                ];
-                
-                for (let selector of indicators) {
-                    if (document.querySelector(selector)) {
-                        return {detected: true, selector: selector};
-                    }
-                }
-                
-                // Check for scroll event listeners
-                const hasScrollListeners = window.getEventListeners && 
-                    Object.keys(window.getEventListeners(window)).includes('scroll');
-                
-                return {
-                    detected: hasScrollListeners || false,
-                    selector: null,
-                    hasScrollListeners: hasScrollListeners
-                };
-            """,
-            "wait_for_content_load": """
-                return new Promise((resolve) => {
-                    let attempts = 0;
-                    const maxAttempts = arguments[0] || 10;
-                    const checkInterval = arguments[1] || 500;
-                    
-                    function checkForNewContent() {
-                        attempts++;
-                        
-                        // Check if loading indicators are gone
-                        const loadingElements = document.querySelectorAll(
-                            '.loading, .spinner, [class*="load"], [aria-label*="load"]'
-                        );
-                        
-                        const stillLoading = Array.from(loadingElements).some(el => 
-                            el.offsetParent !== null && !el.hidden
-                        );
-                        
-                        if (!stillLoading || attempts >= maxAttempts) {
-                            resolve({
-                                attempts: attempts,
-                                stillLoading: stillLoading,
-                                loadingElements: loadingElements.length
-                            });
-                        } else {
-                            setTimeout(checkForNewContent, checkInterval);
-                        }
-                    }
-                    
-                    checkForNewContent();
-                });
-            """
-        }
-    
-    async def perform_virtual_scroll(
-        self,
-        page_executor: Callable,  # Function to execute JS on the page
-        content_extractor: Callable,  # Function to extract current page content
-        url: str = "",
-        **kwargs
-    ) -> VirtualScrollResult:
-        """
-        Perform virtual scrolling with content extraction.
-        
-        Args:
-            page_executor: Function to execute JavaScript on the page
-            content_extractor: Function to extract content from current page state
-            url: URL being scrolled (for logging)
-            **kwargs: Additional parameters
-            
-        Returns:
-            VirtualScrollResult with extracted content and metadata
-        """
-        start_time = time.time()
-        state = ScrollState()
-        content_blocks = []
-        
-        try:
-            logger.info(
-                "virtual_scroll_started",
-                url=url,
-                config=self.config.__dict__
-            )
-            
-            # Auto-detect infinite scroll if enabled
-            if self.config.auto_detect_infinite_scroll:
-                detection_result = await self._detect_infinite_scroll(page_executor)
-                state.is_infinite_scroll_detected = detection_result.get('detected', False)
-                
-                if state.is_infinite_scroll_detected:
-                    logger.info("infinite_scroll_detected", url=url, detection=detection_result)
-            
-            # Initial content extraction
-            initial_content = await content_extractor()
-            if initial_content:
-                state.total_content_length = len(initial_content)
-                state.content_snapshots.append(initial_content[:500])  # Store first 500 chars
-                content_blocks.append({
-                    'scroll_position': 0,
-                    'content': initial_content,
-                    'timestamp': time.time(),
-                    'content_length': len(initial_content)
-                })
-            
-            # Perform scrolling iterations
-            successful_scrolls = 0
-            consecutive_no_change = 0
-            
-            for scroll_iteration in range(self.config.scroll_count):
-                if scroll_iteration >= self.config.max_scroll_attempts:
-                    logger.warning("max_scroll_attempts_reached", url=url)
-                    break
-                
-                # Perform scroll action
-                scroll_result = await self._perform_single_scroll(
-                    page_executor, scroll_iteration
-                )
-                
-                if not scroll_result['success']:
-                    logger.warning(
-                        "scroll_action_failed",
-                        iteration=scroll_iteration,
-                        url=url
-                    )
-                    continue
-                
-                state.current_scroll = scroll_iteration + 1
-                state.scroll_history.append(scroll_result['position'])
-                
-                # Wait for content to load
-                await self._wait_for_content_stabilization(page_executor)
-                
-                # Extract content after scroll
-                new_content = await content_extractor()
-                if new_content:
-                    # Check if content actually changed
-                    content_change = len(new_content) - state.total_content_length
-                    
-                    if content_change >= self.config.min_content_increase:
-                        # Significant content change detected
-                        state.total_content_length = len(new_content)
-                        state.last_content_change = time.time()
-                        state.content_snapshots.append(new_content[-500:])  # Store last 500 chars
-                        consecutive_no_change = 0
-                        successful_scrolls += 1
-                        
-                        content_blocks.append({
-                            'scroll_position': scroll_iteration + 1,
-                            'content': new_content,
-                            'content_delta': content_change,
-                            'timestamp': time.time(),
-                            'content_length': len(new_content)
-                        })
-                        
-                        state.last_successful_scroll = scroll_iteration
-                        
-                        logger.debug(
-                            "scroll_content_updated",
-                            iteration=scroll_iteration,
-                            content_change=content_change,
-                            url=url
-                        )
-                    else:
-                        # No significant content change
-                        consecutive_no_change += 1
-                        
-                        if consecutive_no_change >= 3:
-                            logger.info(
-                                "no_content_change_detected",
-                                consecutive_attempts=consecutive_no_change,
-                                url=url
-                            )
-                            break
-                
-                # Check if we've reached the bottom
-                if await self._check_if_at_bottom(page_executor):
-                    logger.info("reached_bottom_of_page", url=url)
-                    break
-                
-                # Adaptive delay based on content loading
-                await self._adaptive_wait(state, scroll_iteration)
-            
-            # Final content extraction
-            final_content = await content_extractor()
-            
-            # Calculate performance metrics
-            end_time = time.time()
-            performance_metrics = {
-                'total_time': end_time - start_time,
-                'scrolls_performed': state.current_scroll,
-                'successful_scrolls': successful_scrolls,
-                'content_increase_ratio': state.total_content_length / max(1, len(initial_content or "")),
-                'avg_scroll_time': (end_time - start_time) / max(1, state.current_scroll),
-                'content_per_scroll': state.total_content_length / max(1, successful_scrolls),
-                'stabilization_time': self.config.content_stabilization_time
-            }
-            
-            # Prepare scroll metadata
-            scroll_metadata = {
-                'infinite_scroll_detected': state.is_infinite_scroll_detected,
-                'total_scrolls': state.current_scroll,
-                'successful_scrolls': successful_scrolls,
-                'final_content_length': len(final_content) if final_content else 0,
-                'content_snapshots_count': len(state.content_snapshots),
-                'scroll_history': state.scroll_history,
-                'config': self.config.__dict__
-            }
-            
-            logger.info(
-                "virtual_scroll_completed",
-                url=url,
-                total_scrolls=state.current_scroll,
-                successful_scrolls=successful_scrolls,
-                final_content_length=len(final_content) if final_content else 0
-            )
-            
-            return VirtualScrollResult(
-                success=True,
-                total_scrolls=state.current_scroll,
-                final_content=final_content or "",
-                content_blocks=content_blocks,
-                scroll_metadata=scroll_metadata,
-                performance_metrics=performance_metrics
-            )
-            
-        except Exception as e:
-            logger.error("virtual_scroll_failed", url=url, error=str(e))
-            
-            return VirtualScrollResult(
-                success=False,
-                total_scrolls=state.current_scroll,
-                final_content="",
-                content_blocks=content_blocks,
-                scroll_metadata={'error': str(e)},
-                error_message=str(e)
-            )
-    
-    async def _detect_infinite_scroll(self, page_executor: Callable) -> Dict[str, Any]:
-        """Detect if the page uses infinite scrolling."""
-        try:
-            result = await page_executor(self.scroll_js["detect_infinite_scroll"])
-            return result if isinstance(result, dict) else {'detected': False}
-        except Exception as e:
-            logger.warning("infinite_scroll_detection_failed", error=str(e))
-            return {'detected': False, 'error': str(e)}
-    
-    async def _perform_single_scroll(
-        self, 
-        page_executor: Callable, 
-        iteration: int
-    ) -> Dict[str, Any]:
-        """Perform a single scroll action."""
-        try:
-            if self.config.scroll_by == "viewport_height":
-                result = await page_executor(self.scroll_js["viewport_height"])
-            elif self.config.scroll_by == "container_height":
-                result = await page_executor(
-                    self.scroll_js["container_height"],
-                    self.config.container_selector or "body"
-                )
-            else:  # pixels
-                result = await page_executor(
-                    self.scroll_js["pixels"],
-                    self.config.scroll_pixels
-                )
-            
-            return {
-                'success': True,
-                'position': result if isinstance(result, (int, float)) else 0,
-                'iteration': iteration
-            }
-            
-        except Exception as e:
-            logger.error("scroll_action_failed", iteration=iteration, error=str(e))
-            return {'success': False, 'error': str(e), 'iteration': iteration}
-    
-    async def _wait_for_content_stabilization(self, page_executor: Callable):
-        """Wait for content to stabilize after scrolling."""
-        try:
-            # Basic wait
-            await asyncio.sleep(self.config.wait_after_scroll)
-            
-            # If specified, wait for specific selector
-            if self.config.wait_for_selector:
-                # Simple implementation - in production, you'd want more sophisticated waiting
-                await asyncio.sleep(1.0)
-            
-            # Wait for loading indicators to disappear
-            if self.config.scroll_pause_detection:
-                await page_executor(
-                    self.scroll_js["wait_for_content_load"],
-                    10,  # max attempts
-                    500  # check interval ms
-                )
-            
-        except Exception as e:
-            logger.warning("content_stabilization_wait_failed", error=str(e))
-            # Continue with default wait
-            await asyncio.sleep(self.config.wait_after_scroll)
-    
-    async def _check_if_at_bottom(self, page_executor: Callable) -> bool:
-        """Check if we've reached the bottom of the page."""
-        try:
-            result = await page_executor(self.scroll_js["get_scroll_position"])
-            
-            if isinstance(result, dict):
-                scroll_y = result.get('scrollY', 0)
-                scroll_height = result.get('scrollHeight', 0)
-                client_height = result.get('clientHeight', 0)
-                
-                # Check if we're within 100 pixels of the bottom
-                return (scroll_y + client_height) >= (scroll_height - 100)
-            
-            return False
-            
-        except Exception as e:
-            logger.warning("bottom_check_failed", error=str(e))
-            return False
-    
-    async def _adaptive_wait(self, state: ScrollState, iteration: int):
-        """Implement adaptive waiting based on scrolling patterns."""
-        base_wait = self.config.wait_after_scroll
-        
-        # Increase wait time if we haven't seen content changes recently
-        time_since_change = time.time() - state.last_content_change
-        if time_since_change > 10.0:  # 10 seconds without change
-            base_wait *= 1.5
-        
-        # Decrease wait time if we're getting consistent content updates
-        if len(state.content_snapshots) > 3:
-            recent_changes = len(set(state.content_snapshots[-3:]))
-            if recent_changes >= 2:  # Recent content diversity
-                base_wait *= 0.8
-        
-        # Increase wait time for later iterations (content might load slower)
-        if iteration > 10:
-            base_wait *= 1.2
-        
-        # Ensure reasonable bounds
-        final_wait = max(0.5, min(10.0, base_wait))
-        
-        await asyncio.sleep(final_wait)
-
-
-class PuppeteerVirtualScroller:
-    """
-    Virtual scroller that integrates with Puppeteer service.
-    
-    This class provides a bridge between the virtual scrolling logic
-    and the Puppeteer-based browser automation.
-    """
-    
-    def __init__(self, puppeteer_service_url: str):
-        """Initialize Puppeteer virtual scroller."""
-        self.service_url = puppeteer_service_url
-        self.handler = None
-    
-    async def scroll_and_extract(
-        self,
-        url: str,
-        config: VirtualScrollConfig,
-        headers: Optional[Dict[str, str]] = None,
-        **kwargs
-    ) -> VirtualScrollResult:
-        """
-        Scroll page and extract content using Puppeteer service.
-        
-        Args:
-            url: URL to scroll
-            config: Virtual scrolling configuration
-            headers: Optional HTTP headers
-            **kwargs: Additional parameters
-            
-        Returns:
-            VirtualScrollResult with extracted content
-        """
-        self.handler = VirtualScrollHandler(config)
-        
-        # Create page executor for Puppeteer
-        async def page_executor(js_code: str, *args) -> Any:
-            return await self._execute_js_on_puppeteer(url, js_code, args, headers)
-        
-        # Create content extractor for current page state
-        async def content_extractor() -> str:
-            return await self._extract_page_content(url, headers)
-        
-        return await self.handler.perform_virtual_scroll(
-            page_executor=page_executor,
-            content_extractor=content_extractor,
-            url=url,
-            **kwargs
-        )
-    
-    async def _execute_js_on_puppeteer(
-        self,
-        url: str,
-        js_code: str,
-        args: Tuple,
-        headers: Optional[Dict[str, str]]
-    ) -> Any:
-        """Execute JavaScript on Puppeteer service."""
-        # This would integrate with your existing Puppeteer service
-        # For now, this is a placeholder implementation
-        import httpx
-        
-        async with httpx.AsyncClient() as client:
-            payload = {
-                'url': url,
-                'javascript': js_code,
-                'args': list(args) if args else [],
-                'headers': headers or {},
-                'waitUntil': 'networkidle0'
-            }
-            
-            response = await client.post(
-                f"{self.service_url}/execute-js",
-                json=payload,
-                timeout=30.0
-            )
-            
-            if response.status_code == 200:
-                result = response.json()
-                return result.get('result')
-            else:
-                raise Exception(f"Puppeteer service error: {response.status_code}")
-    
-    async def _extract_page_content(
-        self,
-        url: str,
-        headers: Optional[Dict[str, str]]
-    ) -> str:
-        """Extract current page content via Puppeteer service."""
-        # This would integrate with your existing Puppeteer service
-        import httpx
-        
-        async with httpx.AsyncClient() as client:
-            payload = {
-                'url': url,
-                'headers': headers or {},
-                'waitUntil': 'networkidle0'
-            }
-            
-            response = await client.post(
-                f"{self.service_url}/render",
-                json=payload,
-                timeout=30.0
-            )
-            
-            if response.status_code == 200:
-                result = response.json()
-                return result.get('html', '')
-            else:
-                return ''
-
-
-# Convenience functions
-def create_virtual_scroll_config(
-    container_selector: Optional[str] = None,
-    scroll_count: int = 10,
-    wait_after_scroll: float = 2.0,
-    auto_detect: bool = True
-) -> VirtualScrollConfig:
-    """Create virtual scroll configuration with common settings."""
-    return VirtualScrollConfig(
-        container_selector=container_selector,
-        scroll_count=scroll_count,
-        wait_after_scroll=wait_after_scroll,
-        auto_detect_infinite_scroll=auto_detect,
-        check_content_changes=True,
-        scroll_pause_detection=True
-    )
-
-
-async def scroll_infinite_page(
-    url: str,
-    puppeteer_service_url: str,
-    scroll_count: int = 10,
-    wait_time: float = 2.0,
-    container_selector: Optional[str] = None
-) -> VirtualScrollResult:
-    """
-    Convenience function to scroll an infinite page.
-    
-    Args:
-        url: URL to scroll
-        puppeteer_service_url: Puppeteer service endpoint
-        scroll_count: Number of scrolls to perform
-        wait_time: Time to wait after each scroll
-        container_selector: Optional container selector
-        
-    Returns:
-        VirtualScrollResult with extracted content
-    """
-    config = VirtualScrollConfig(
-        container_selector=container_selector,
-        scroll_count=scroll_count,
-        wait_after_scroll=wait_time,
-        auto_detect_infinite_scroll=True
-    )
-    
-    scroller = PuppeteerVirtualScroller(puppeteer_service_url)
-    return await scroller.scroll_and_extract(url, config)
diff --git a/apps/backend/app/services/webhook_integration.py b/apps/backend/app/services/webhook_integration.py
deleted file mode 100644
index e662372..0000000
--- a/apps/backend/app/services/webhook_integration.py
+++ /dev/null
@@ -1,662 +0,0 @@
-"""
-Webhook Integration service inspired by Firecrawl.
-
-Provides comprehensive webhook notifications:
-- Job status change notifications
-- Real-time progress updates
-- Error and failure notifications
-- Customizable webhook payloads
-- Retry logic and failure handling
-- Webhook security and validation
-"""
-
-import asyncio
-import time
-import json
-import hmac
-import hashlib
-from typing import Dict, List, Optional, Any, Callable, Union
-from dataclasses import dataclass, field
-from datetime import datetime, timedelta
-from enum import Enum
-import structlog
-import httpx
-
-from app.config import get_settings
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class WebhookEvent(Enum):
-    """Types of webhook events."""
-    # Job lifecycle events
-    JOB_STARTED = "job.started"
-    JOB_COMPLETED = "job.completed"
-    JOB_FAILED = "job.failed"
-    JOB_CANCELLED = "job.cancelled"
-    JOB_PAUSED = "job.paused"
-    JOB_RESUMED = "job.resumed"
-    
-    # Progress events
-    PROGRESS_UPDATE = "progress.update"
-    MILESTONE_REACHED = "milestone.reached"
-    
-    # Error events
-    ERROR_OCCURRED = "error.occurred"
-    RETRY_ATTEMPTED = "retry.attempted"
-    CRITICAL_ERROR = "error.critical"
-    
-    # Data events
-    DATA_EXTRACTED = "data.extracted"
-    CHANGE_DETECTED = "change.detected"
-    BATCH_COMPLETED = "batch.completed"
-    
-    # System events
-    SYSTEM_ALERT = "system.alert"
-    QUOTA_WARNING = "quota.warning"
-    RATE_LIMIT = "rate_limit.exceeded"
-
-
-class WebhookStatus(Enum):
-    """Status of webhook delivery."""
-    PENDING = "pending"
-    SENT = "sent"
-    FAILED = "failed"
-    RETRYING = "retrying"
-    ABANDONED = "abandoned"
-
-
-@dataclass
-class WebhookConfig:
-    """Configuration for webhook endpoints."""
-    url: str
-    events: List[WebhookEvent] = field(default_factory=list)
-    secret: Optional[str] = None
-    headers: Dict[str, str] = field(default_factory=dict)
-    timeout: int = 30
-    max_retries: int = 3
-    retry_delay: float = 1.0
-    retry_backoff: float = 2.0
-    verify_ssl: bool = True
-    enabled: bool = True
-    
-    # Filtering options
-    job_types: List[str] = field(default_factory=list)  # Filter by job types
-    tags: List[str] = field(default_factory=list)  # Filter by tags
-    
-    # Custom payload options
-    include_data: bool = False
-    include_errors: bool = True
-    include_progress: bool = True
-    max_payload_size: int = 1024 * 1024  # 1MB
-
-
-@dataclass
-class WebhookPayload:
-    """Webhook payload structure."""
-    event: WebhookEvent
-    timestamp: datetime
-    data: Dict[str, Any]
-    source: str = "unsearch_backend"
-    version: str = "1.0"
-    
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary."""
-        return {
-            "event": self.event.value,
-            "timestamp": self.timestamp.isoformat(),
-            "source": self.source,
-            "version": self.version,
-            "data": self.data
-        }
-
-
-@dataclass
-class WebhookAttempt:
-    """Record of webhook delivery attempt."""
-    id: str
-    webhook_config: WebhookConfig
-    payload: WebhookPayload
-    status: WebhookStatus = WebhookStatus.PENDING
-    attempts: int = 0
-    created_at: datetime = field(default_factory=datetime.utcnow)
-    last_attempt_at: Optional[datetime] = None
-    next_attempt_at: Optional[datetime] = None
-    response_status: Optional[int] = None
-    response_body: Optional[str] = None
-    error: Optional[str] = None
-
-
-class WebhookManager:
-    """
-    Comprehensive webhook management service.
-    
-    Provides webhook notifications for various system events:
-    - Job lifecycle management
-    - Real-time progress updates
-    - Error and failure notifications
-    - Custom event handling
-    """
-    
-    def __init__(self):
-        """Initialize webhook manager."""
-        self.webhook_configs: Dict[str, WebhookConfig] = {}
-        self.pending_attempts: Dict[str, WebhookAttempt] = {}
-        self.event_handlers: Dict[WebhookEvent, List[Callable]] = {}
-        
-        self.stats = {
-            "total_webhooks": 0,
-            "successful_deliveries": 0,
-            "failed_deliveries": 0,
-            "active_configs": 0,
-            "events_processed": 0
-        }
-        
-        # Start background worker
-        self._worker_task = None
-        asyncio.create_task(self._start_webhook_worker())
-    
-    def add_webhook_config(
-        self, 
-        config_id: str, 
-        config: WebhookConfig
-    ) -> bool:
-        """Add webhook configuration."""
-        self.webhook_configs[config_id] = config
-        self.stats["active_configs"] = len(self.webhook_configs)
-        
-        logger.info("webhook_config_added",
-                   config_id=config_id,
-                   url=config.url,
-                   events=len(config.events))
-        
-        return True
-    
-    def remove_webhook_config(self, config_id: str) -> bool:
-        """Remove webhook configuration."""
-        if config_id not in self.webhook_configs:
-            return False
-        
-        del self.webhook_configs[config_id]
-        self.stats["active_configs"] = len(self.webhook_configs)
-        
-        logger.info("webhook_config_removed", config_id=config_id)
-        return True
-    
-    def get_webhook_config(self, config_id: str) -> Optional[WebhookConfig]:
-        """Get webhook configuration."""
-        return self.webhook_configs.get(config_id)
-    
-    def list_webhook_configs(self) -> Dict[str, WebhookConfig]:
-        """List all webhook configurations."""
-        return self.webhook_configs.copy()
-    
-    async def send_webhook(
-        self,
-        event: WebhookEvent,
-        data: Dict[str, Any],
-        config_id: Optional[str] = None,
-        tags: Optional[List[str]] = None
-    ) -> List[str]:
-        """
-        Send webhook notification for event.
-        
-        Args:
-            event: Webhook event type
-            data: Event data payload
-            config_id: Specific config to use (if None, uses all matching)
-            tags: Event tags for filtering
-            
-        Returns:
-            List of attempt IDs
-        """
-        tags = tags or []
-        attempt_ids = []
-        
-        # Determine which configs to use
-        configs_to_use = {}
-        
-        if config_id:
-            if config_id in self.webhook_configs:
-                configs_to_use[config_id] = self.webhook_configs[config_id]
-        else:
-            configs_to_use = self.webhook_configs.copy()
-        
-        # Filter configs based on event and tags
-        filtered_configs = {}
-        for cid, config in configs_to_use.items():
-            if not config.enabled:
-                continue
-            
-            # Check if event is subscribed
-            if config.events and event not in config.events:
-                continue
-            
-            # Check tag filtering
-            if config.tags and not any(tag in config.tags for tag in tags):
-                continue
-            
-            filtered_configs[cid] = config
-        
-        # Create webhook attempts
-        for cid, config in filtered_configs.items():
-            attempt_id = await self._create_webhook_attempt(config, event, data)
-            attempt_ids.append(attempt_id)
-        
-        self.stats["events_processed"] += 1
-        self.stats["total_webhooks"] += len(attempt_ids)
-        
-        logger.info("webhook_event_sent",
-                   event=event.value,
-                   configs_matched=len(filtered_configs),
-                   attempts_created=len(attempt_ids))
-        
-        return attempt_ids
-    
-    async def _create_webhook_attempt(
-        self,
-        config: WebhookConfig,
-        event: WebhookEvent,
-        data: Dict[str, Any]
-    ) -> str:
-        """Create a webhook delivery attempt."""
-        # Create payload
-        payload = WebhookPayload(
-            event=event,
-            timestamp=datetime.utcnow(),
-            data=self._prepare_payload_data(data, config)
-        )
-        
-        # Create attempt record
-        attempt_id = f"webhook_{int(time.time() * 1000)}_{id(config)}"
-        attempt = WebhookAttempt(
-            id=attempt_id,
-            webhook_config=config,
-            payload=payload,
-            next_attempt_at=datetime.utcnow()
-        )
-        
-        self.pending_attempts[attempt_id] = attempt
-        
-        return attempt_id
-    
-    def _prepare_payload_data(self, data: Dict[str, Any], config: WebhookConfig) -> Dict[str, Any]:
-        """Prepare payload data according to config."""
-        filtered_data = data.copy()
-        
-        # Remove sensitive data if not requested
-        if not config.include_data:
-            # Remove large data fields
-            for key in ["results", "scraped_content", "extracted_data"]:
-                if key in filtered_data:
-                    if isinstance(filtered_data[key], list):
-                        filtered_data[key] = {"count": len(filtered_data[key])}
-                    else:
-                        filtered_data[key] = {"size": len(str(filtered_data[key]))}
-        
-        if not config.include_errors:
-            filtered_data.pop("errors", None)
-            filtered_data.pop("error", None)
-        
-        if not config.include_progress:
-            filtered_data.pop("progress", None)
-        
-        # Check payload size
-        payload_str = json.dumps(filtered_data)
-        if len(payload_str) > config.max_payload_size:
-            # Truncate large fields
-            filtered_data = self._truncate_payload(filtered_data, config.max_payload_size)
-        
-        return filtered_data
-    
-    def _truncate_payload(self, data: Dict[str, Any], max_size: int) -> Dict[str, Any]:
-        """Truncate payload to fit size limit."""
-        truncated = data.copy()
-        
-        # Truncate large text fields
-        for key, value in data.items():
-            if isinstance(value, str) and len(value) > 1000:
-                truncated[key] = value[:1000] + "... [truncated]"
-            elif isinstance(value, list) and len(value) > 10:
-                truncated[key] = value[:10] + [{"truncated": f"{len(value) - 10} more items"}]
-        
-        return truncated
-    
-    async def _start_webhook_worker(self):
-        """Start background worker for webhook delivery."""
-        self._worker_task = asyncio.create_task(self._webhook_worker())
-    
-    async def _webhook_worker(self):
-        """Background worker that delivers webhooks."""
-        while True:
-            try:
-                await self._process_pending_webhooks()
-                await asyncio.sleep(5)  # Check every 5 seconds
-                
-            except Exception as e:
-                logger.error("webhook_worker_error", error=str(e))
-                await asyncio.sleep(30)  # Wait 30 seconds before retrying
-    
-    async def _process_pending_webhooks(self):
-        """Process pending webhook deliveries."""
-        now = datetime.utcnow()
-        ready_attempts = []
-        
-        # Find attempts ready for delivery
-        for attempt_id, attempt in self.pending_attempts.items():
-            if attempt.status == WebhookStatus.PENDING or (
-                attempt.status == WebhookStatus.RETRYING and 
-                attempt.next_attempt_at and 
-                now >= attempt.next_attempt_at
-            ):
-                ready_attempts.append(attempt)
-        
-        # Process attempts concurrently
-        if ready_attempts:
-            await asyncio.gather(
-                *[self._deliver_webhook(attempt) for attempt in ready_attempts],
-                return_exceptions=True
-            )
-    
-    async def _deliver_webhook(self, attempt: WebhookAttempt):
-        """Deliver a single webhook."""
-        config = attempt.webhook_config
-        
-        try:
-            attempt.attempts += 1
-            attempt.last_attempt_at = datetime.utcnow()
-            attempt.status = WebhookStatus.SENT if attempt.attempts == 1 else WebhookStatus.RETRYING
-            
-            # Prepare request
-            payload_dict = attempt.payload.to_dict()
-            headers = config.headers.copy()
-            headers["Content-Type"] = "application/json"
-            headers["User-Agent"] = "UnSearch-Webhook/1.0"
-            
-            # Add signature if secret is provided
-            if config.secret:
-                signature = self._generate_signature(json.dumps(payload_dict), config.secret)
-                headers["X-Webhook-Signature"] = signature
-            
-            # Make HTTP request
-            async with httpx.AsyncClient(verify=config.verify_ssl) as client:
-                response = await client.post(
-                    config.url,
-                    json=payload_dict,
-                    headers=headers,
-                    timeout=config.timeout
-                )
-                
-                attempt.response_status = response.status_code
-                attempt.response_body = response.text[:1000]  # Limit response body
-                
-                if 200 <= response.status_code < 300:
-                    # Success
-                    attempt.status = WebhookStatus.SENT
-                    self.stats["successful_deliveries"] += 1
-                    
-                    # Remove from pending
-                    if attempt.id in self.pending_attempts:
-                        del self.pending_attempts[attempt.id]
-                    
-                    logger.info("webhook_delivered_successfully",
-                               attempt_id=attempt.id,
-                               url=config.url,
-                               status_code=response.status_code,
-                               attempts=attempt.attempts)
-                else:
-                    # HTTP error
-                    raise httpx.HTTPStatusError(
-                        message=f"HTTP {response.status_code}",
-                        request=response.request,
-                        response=response
-                    )
-        
-        except Exception as e:
-            await self._handle_webhook_failure(attempt, str(e))
-    
-    async def _handle_webhook_failure(self, attempt: WebhookAttempt, error: str):
-        """Handle webhook delivery failure."""
-        config = attempt.webhook_config
-        attempt.error = error
-        
-        if attempt.attempts >= config.max_retries:
-            # Max retries reached, abandon
-            attempt.status = WebhookStatus.ABANDONED
-            self.stats["failed_deliveries"] += 1
-            
-            # Remove from pending
-            if attempt.id in self.pending_attempts:
-                del self.pending_attempts[attempt.id]
-            
-            logger.warning("webhook_delivery_abandoned",
-                          attempt_id=attempt.id,
-                          url=config.url,
-                          attempts=attempt.attempts,
-                          error=error)
-        else:
-            # Schedule retry
-            attempt.status = WebhookStatus.FAILED
-            retry_delay = config.retry_delay * (config.retry_backoff ** (attempt.attempts - 1))
-            attempt.next_attempt_at = datetime.utcnow() + timedelta(seconds=retry_delay)
-            
-            logger.info("webhook_delivery_failed_will_retry",
-                       attempt_id=attempt.id,
-                       url=config.url,
-                       attempts=attempt.attempts,
-                       retry_in_seconds=retry_delay,
-                       error=error)
-    
-    def _generate_signature(self, payload: str, secret: str) -> str:
-        """Generate HMAC signature for webhook payload."""
-        signature = hmac.new(
-            secret.encode('utf-8'),
-            payload.encode('utf-8'),
-            hashlib.sha256
-        ).hexdigest()
-        return f"sha256={signature}"
-    
-    def register_event_handler(
-        self,
-        event: WebhookEvent,
-        handler: Callable[[Dict[str, Any]], Any]
-    ):
-        """Register custom event handler."""
-        if event not in self.event_handlers:
-            self.event_handlers[event] = []
-        
-        self.event_handlers[event].append(handler)
-        logger.info("event_handler_registered", event=event.value)
-    
-    async def emit_event(
-        self,
-        event: WebhookEvent,
-        data: Dict[str, Any],
-        tags: Optional[List[str]] = None
-    ) -> List[str]:
-        """
-        Emit event to both webhooks and custom handlers.
-        
-        Args:
-            event: Event type
-            data: Event data
-            tags: Event tags
-            
-        Returns:
-            List of webhook attempt IDs
-        """
-        # Call custom handlers
-        if event in self.event_handlers:
-            for handler in self.event_handlers[event]:
-                try:
-                    handler(data)
-                except Exception as e:
-                    logger.error("event_handler_failed",
-                               event=event.value,
-                               error=str(e))
-        
-        # Send webhooks
-        return await self.send_webhook(event, data, tags=tags)
-    
-    async def get_webhook_stats(self) -> Dict[str, Any]:
-        """Get webhook service statistics."""
-        pending_count = len(self.pending_attempts)
-        failed_count = len([a for a in self.pending_attempts.values() if a.status == WebhookStatus.FAILED])
-        
-        return {
-            "webhook_stats": self.stats,
-            "active_configs": len(self.webhook_configs),
-            "pending_attempts": pending_count,
-            "failed_attempts": failed_count,
-            "success_rate": (
-                self.stats["successful_deliveries"] / 
-                max(1, self.stats["total_webhooks"])
-            ) if self.stats["total_webhooks"] > 0 else 0,
-            "worker_running": self._worker_task is not None and not self._worker_task.done()
-        }
-    
-    async def get_webhook_attempt_status(self, attempt_id: str) -> Optional[Dict[str, Any]]:
-        """Get status of a webhook attempt."""
-        if attempt_id not in self.pending_attempts:
-            return None
-        
-        attempt = self.pending_attempts[attempt_id]
-        
-        return {
-            "id": attempt.id,
-            "status": attempt.status.value,
-            "attempts": attempt.attempts,
-            "created_at": attempt.created_at.isoformat(),
-            "last_attempt_at": attempt.last_attempt_at.isoformat() if attempt.last_attempt_at else None,
-            "next_attempt_at": attempt.next_attempt_at.isoformat() if attempt.next_attempt_at else None,
-            "response_status": attempt.response_status,
-            "error": attempt.error,
-            "webhook_url": attempt.webhook_config.url,
-            "event": attempt.payload.event.value
-        }
-    
-    async def cleanup(self):
-        """Cleanup resources."""
-        if self._worker_task:
-            self._worker_task.cancel()
-            try:
-                await self._worker_task
-            except asyncio.CancelledError:
-                pass
-
-
-# Singleton service
-_webhook_manager: Optional[WebhookManager] = None
-
-
-async def get_webhook_manager() -> WebhookManager:
-    """Get or create webhook manager service instance."""
-    global _webhook_manager
-    
-    if _webhook_manager is None:
-        _webhook_manager = WebhookManager()
-    
-    return _webhook_manager
-
-
-# Convenience functions
-async def send_job_webhook(
-    job_id: str,
-    event: str,
-    status: str,
-    data: Optional[Dict[str, Any]] = None,
-    webhook_url: Optional[str] = None
-) -> List[str]:
-    """
-    Send job-related webhook notification.
-    
-    Args:
-        job_id: Job identifier
-        event: Event type (started, completed, failed, etc.)
-        status: Current job status
-        data: Additional job data
-        webhook_url: Specific webhook URL to use
-        
-    Returns:
-        List of webhook attempt IDs
-    """
-    manager = await get_webhook_manager()
-    
-    # Map event string to enum
-    event_mapping = {
-        "started": WebhookEvent.JOB_STARTED,
-        "completed": WebhookEvent.JOB_COMPLETED,
-        "failed": WebhookEvent.JOB_FAILED,
-        "cancelled": WebhookEvent.JOB_CANCELLED,
-        "paused": WebhookEvent.JOB_PAUSED,
-        "resumed": WebhookEvent.JOB_RESUMED
-    }
-    
-    webhook_event = event_mapping.get(event, WebhookEvent.JOB_STARTED)
-    
-    payload_data = {
-        "job_id": job_id,
-        "status": status,
-        **(data or {})
-    }
-    
-    # If specific webhook URL provided, create temporary config
-    if webhook_url:
-        temp_config_id = f"temp_{job_id}_{int(time.time())}"
-        temp_config = WebhookConfig(
-            url=webhook_url,
-            events=[webhook_event],
-            timeout=30,
-            max_retries=3
-        )
-        
-        manager.add_webhook_config(temp_config_id, temp_config)
-        
-        try:
-            return await manager.send_webhook(webhook_event, payload_data, temp_config_id)
-        finally:
-            manager.remove_webhook_config(temp_config_id)
-    else:
-        return await manager.send_webhook(webhook_event, payload_data)
-
-
-async def send_progress_webhook(
-    job_id: str,
-    progress_data: Dict[str, Any],
-    tags: Optional[List[str]] = None
-) -> List[str]:
-    """Send progress update webhook."""
-    manager = await get_webhook_manager()
-    
-    payload_data = {
-        "job_id": job_id,
-        "progress": progress_data
-    }
-    
-    return await manager.send_webhook(WebhookEvent.PROGRESS_UPDATE, payload_data, tags=tags)
-
-
-async def send_error_webhook(
-    error_type: str,
-    error_message: str,
-    context: Optional[Dict[str, Any]] = None,
-    critical: bool = False
-) -> List[str]:
-    """Send error notification webhook."""
-    manager = await get_webhook_manager()
-    
-    event = WebhookEvent.CRITICAL_ERROR if critical else WebhookEvent.ERROR_OCCURRED
-    
-    payload_data = {
-        "error_type": error_type,
-        "error_message": error_message,
-        "critical": critical,
-        "context": context or {}
-    }
-    
-    return await manager.send_webhook(event, payload_data)
-
-
-
-
diff --git a/apps/backend/app/services/website_mapping.py b/apps/backend/app/services/website_mapping.py
deleted file mode 100644
index dab1797..0000000
--- a/apps/backend/app/services/website_mapping.py
+++ /dev/null
@@ -1,772 +0,0 @@
-"""
-Website mapping service for fast URL discovery inspired by Firecrawl.
-
-Provides comprehensive website mapping capabilities:
-- Sitemap-based URL discovery
-- Search engine-based URL mapping
-- Subdomain and path filtering
-- External link detection
-- Fast URL enumeration
-"""
-
-import asyncio
-import time
-import re
-import xml.etree.ElementTree as ET
-from typing import Dict, List, Optional, Any, Set, Union
-from urllib.parse import urlparse, urljoin, quote
-from dataclasses import dataclass, field
-from enum import Enum
-import structlog
-import httpx
-
-from app.config import get_settings
-from app.services.multi_search import get_multi_search_service, SearchOptions
-from app.utils.text_processing import sanitize_text
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class MapStrategy(Enum):
-    """Website mapping strategies."""
-    SITEMAP_ONLY = "sitemap_only"
-    SEARCH_ENGINE = "search_engine"
-    COMBINED = "combined"
-    CRAWL_BASED = "crawl_based"
-
-
-@dataclass
-class MapOptions:
-    """Configuration for website mapping."""
-    strategy: MapStrategy = MapStrategy.COMBINED
-    limit: int = 1000
-    include_subdomains: bool = True
-    allow_external_links: bool = False
-    ignore_sitemap: bool = False
-    filter_by_path: bool = True
-    search_query: Optional[str] = None
-    timeout: int = 30
-    max_depth: int = 3
-    concurrent_requests: int = 10
-    
-    # Sitemap options
-    follow_sitemap_index: bool = True
-    sitemap_timeout: int = 15
-    
-    # Search engine options
-    search_results_per_page: int = 100
-    max_search_pages: int = 10
-    
-    # Filtering options
-    include_patterns: List[str] = field(default_factory=list)
-    exclude_patterns: List[str] = field(default_factory=list)
-
-
-@dataclass
-class DiscoveredURL:
-    """Represents a discovered URL with metadata."""
-    url: str
-    source: str  # "sitemap", "search", "crawl", "index"
-    title: Optional[str] = None
-    description: Optional[str] = None
-    last_modified: Optional[str] = None
-    change_frequency: Optional[str] = None
-    priority: Optional[float] = None
-    content_type: Optional[str] = None
-    status_code: Optional[int] = None
-    discovery_time: float = field(default_factory=time.time)
-    parent_url: Optional[str] = None
-    depth: int = 0
-
-
-@dataclass
-class WebsiteMapResult:
-    """Result of website mapping operation."""
-    base_url: str
-    discovered_urls: List[DiscoveredURL]
-    total_urls: int
-    sources_breakdown: Dict[str, int]
-    processing_time_ms: int
-    success: bool
-    error: Optional[str] = None
-    metadata: Dict[str, Any] = field(default_factory=dict)
-
-
-class WebsiteMapper:
-    """
-    Comprehensive website mapping service.
-    
-    Discovers URLs from websites using multiple strategies:
-    - Sitemap parsing (XML sitemaps, robots.txt)
-    - Search engine queries (site: operator)
-    - Link crawling and discovery
-    - Index-based lookups
-    """
-    
-    def __init__(self):
-        """Initialize website mapper."""
-        self.client = httpx.AsyncClient(
-            timeout=httpx.Timeout(30),
-            limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
-            headers={
-                "User-Agent": settings.scraping_user_agent
-            }
-        )
-        self.discovered_cache: Dict[str, List[DiscoveredURL]] = {}
-        self.mapping_stats = {"total_requests": 0, "successful_maps": 0, "urls_discovered": 0}
-    
-    async def map_website(
-        self, 
-        url: str, 
-        options: Optional[MapOptions] = None
-    ) -> WebsiteMapResult:
-        """
-        Map a website to discover all accessible URLs.
-        
-        Args:
-            url: Base URL to map
-            options: Mapping configuration options
-            
-        Returns:
-            WebsiteMapResult with discovered URLs and metadata
-        """
-        start_time = time.time()
-        options = options or MapOptions()
-        
-        self.mapping_stats["total_requests"] += 1
-        
-        logger.info("website_mapping_started", 
-                   url=url, 
-                   strategy=options.strategy.value,
-                   limit=options.limit)
-        
-        try:
-            discovered_urls: List[DiscoveredURL] = []
-            sources_breakdown = {"sitemap": 0, "search": 0, "crawl": 0, "index": 0}
-            
-            # Execute mapping strategy
-            if options.strategy == MapStrategy.SITEMAP_ONLY:
-                sitemap_urls = await self._discover_from_sitemaps(url, options)
-                discovered_urls.extend(sitemap_urls)
-                sources_breakdown["sitemap"] = len(sitemap_urls)
-                
-            elif options.strategy == MapStrategy.SEARCH_ENGINE:
-                search_urls = await self._discover_from_search_engines(url, options)
-                discovered_urls.extend(search_urls)
-                sources_breakdown["search"] = len(search_urls)
-                
-            elif options.strategy == MapStrategy.CRAWL_BASED:
-                crawl_urls = await self._discover_from_crawling(url, options)
-                discovered_urls.extend(crawl_urls)
-                sources_breakdown["crawl"] = len(crawl_urls)
-                
-            else:  # COMBINED strategy
-                # Run multiple discovery methods in parallel
-                sitemap_task = self._discover_from_sitemaps(url, options)
-                search_task = self._discover_from_search_engines(url, options)
-                index_task = self._discover_from_index(url, options)
-                
-                sitemap_urls, search_urls, index_urls = await asyncio.gather(
-                    sitemap_task, search_task, index_task, return_exceptions=True
-                )
-                
-                # Handle results and exceptions
-                if isinstance(sitemap_urls, list):
-                    discovered_urls.extend(sitemap_urls)
-                    sources_breakdown["sitemap"] = len(sitemap_urls)
-                
-                if isinstance(search_urls, list):
-                    discovered_urls.extend(search_urls)
-                    sources_breakdown["search"] = len(search_urls)
-                
-                if isinstance(index_urls, list):
-                    discovered_urls.extend(index_urls)
-                    sources_breakdown["index"] = len(index_urls)
-            
-            # Remove duplicates and apply filtering
-            unique_urls = await self._deduplicate_and_filter(discovered_urls, url, options)
-            
-            # Apply limits
-            final_urls = unique_urls[:options.limit]
-            
-            processing_time_ms = int((time.time() - start_time) * 1000)
-            
-            # Update stats
-            self.mapping_stats["successful_maps"] += 1
-            self.mapping_stats["urls_discovered"] += len(final_urls)
-            
-            # Cache results
-            cache_key = f"{url}:{hash(str(options))}"
-            self.discovered_cache[cache_key] = final_urls
-            
-            result = WebsiteMapResult(
-                base_url=url,
-                discovered_urls=final_urls,
-                total_urls=len(final_urls),
-                sources_breakdown=sources_breakdown,
-                processing_time_ms=processing_time_ms,
-                success=True,
-                metadata={
-                    "strategy_used": options.strategy.value,
-                    "original_discovered": len(discovered_urls),
-                    "after_deduplication": len(unique_urls),
-                    "after_filtering": len(final_urls)
-                }
-            )
-            
-            logger.info("website_mapping_completed",
-                       url=url,
-                       total_urls=len(final_urls),
-                       sources=sources_breakdown,
-                       processing_time_ms=processing_time_ms)
-            
-            return result
-            
-        except Exception as e:
-            processing_time_ms = int((time.time() - start_time) * 1000)
-            error_msg = str(e)
-            
-            logger.error("website_mapping_failed", 
-                        url=url, 
-                        error=error_msg,
-                        processing_time_ms=processing_time_ms)
-            
-            return WebsiteMapResult(
-                base_url=url,
-                discovered_urls=[],
-                total_urls=0,
-                sources_breakdown={},
-                processing_time_ms=processing_time_ms,
-                success=False,
-                error=error_msg
-            )
-    
-    async def _discover_from_sitemaps(
-        self, 
-        base_url: str, 
-        options: MapOptions
-    ) -> List[DiscoveredURL]:
-        """Discover URLs from XML sitemaps."""
-        if options.ignore_sitemap:
-            return []
-        
-        discovered_urls = []
-        
-        try:
-            # Try common sitemap locations
-            sitemap_urls = [
-                urljoin(base_url, "/sitemap.xml"),
-                urljoin(base_url, "/sitemap_index.xml"),
-                urljoin(base_url, "/sitemaps.xml"),
-                urljoin(base_url, "/sitemap/index.xml"),
-                urljoin(base_url, "/wp-sitemap.xml"),  # WordPress
-            ]
-            
-            # Also check robots.txt for sitemap declarations
-            robots_sitemaps = await self._extract_sitemaps_from_robots(base_url)
-            sitemap_urls.extend(robots_sitemaps)
-            
-            # Remove duplicates
-            sitemap_urls = list(set(sitemap_urls))
-            
-            # Process sitemaps concurrently
-            semaphore = asyncio.Semaphore(options.concurrent_requests)
-            
-            async def process_sitemap(sitemap_url: str):
-                async with semaphore:
-                    return await self._parse_single_sitemap(sitemap_url, options)
-            
-            # Process all sitemaps
-            sitemap_results = await asyncio.gather(
-                *[process_sitemap(url) for url in sitemap_urls],
-                return_exceptions=True
-            )
-            
-            # Collect URLs from successful sitemap parses
-            for result in sitemap_results:
-                if isinstance(result, list):
-                    discovered_urls.extend(result)
-                    if len(discovered_urls) >= options.limit * 2:  # Stop if we have enough
-                        break
-            
-            logger.info("sitemap_discovery_completed",
-                       base_url=base_url,
-                       sitemaps_processed=len(sitemap_urls),
-                       urls_discovered=len(discovered_urls))
-            
-            return discovered_urls[:options.limit]
-            
-        except Exception as e:
-            logger.error("sitemap_discovery_failed", base_url=base_url, error=str(e))
-            return []
-    
-    async def _parse_single_sitemap(
-        self, 
-        sitemap_url: str, 
-        options: MapOptions
-    ) -> List[DiscoveredURL]:
-        """Parse a single XML sitemap."""
-        try:
-            response = await self.client.get(sitemap_url, timeout=options.sitemap_timeout)
-            if response.status_code != 200:
-                return []
-            
-            content = response.text
-            root = ET.fromstring(content)
-            
-            # Handle namespaces
-            namespace = ""
-            if root.tag.startswith("{"):
-                namespace = root.tag.split("}")[0] + "}"
-            
-            discovered_urls = []
-            
-            # Check if this is a sitemap index
-            sitemap_elements = root.findall(f"{namespace}sitemap")
-            if sitemap_elements and options.follow_sitemap_index:
-                # Recursively process child sitemaps
-                for sitemap_elem in sitemap_elements:
-                    loc_elem = sitemap_elem.find(f"{namespace}loc")
-                    if loc_elem is not None:
-                        child_sitemap_url = loc_elem.text.strip()
-                        child_urls = await self._parse_single_sitemap(child_sitemap_url, options)
-                        discovered_urls.extend(child_urls)
-            
-            else:
-                # Parse regular sitemap URLs
-                url_elements = root.findall(f"{namespace}url")
-                for url_elem in url_elements:
-                    loc_elem = url_elem.find(f"{namespace}loc")
-                    if loc_elem is not None:
-                        url = loc_elem.text.strip()
-                        
-                        # Extract additional metadata
-                        lastmod_elem = url_elem.find(f"{namespace}lastmod")
-                        changefreq_elem = url_elem.find(f"{namespace}changefreq")
-                        priority_elem = url_elem.find(f"{namespace}priority")
-                        
-                        discovered_url = DiscoveredURL(
-                            url=url,
-                            source="sitemap",
-                            last_modified=lastmod_elem.text.strip() if lastmod_elem is not None else None,
-                            change_frequency=changefreq_elem.text.strip() if changefreq_elem is not None else None,
-                            priority=float(priority_elem.text.strip()) if priority_elem is not None else None
-                        )
-                        
-                        discovered_urls.append(discovered_url)
-            
-            return discovered_urls
-            
-        except Exception as e:
-            logger.warning("sitemap_parsing_failed", sitemap_url=sitemap_url, error=str(e))
-            return []
-    
-    async def _extract_sitemaps_from_robots(self, base_url: str) -> List[str]:
-        """Extract sitemap URLs from robots.txt."""
-        try:
-            robots_url = urljoin(base_url, "/robots.txt")
-            response = await self.client.get(robots_url, timeout=10)
-            
-            if response.status_code != 200:
-                return []
-            
-            sitemaps = []
-            for line in response.text.split('\n'):
-                line = line.strip()
-                if line.lower().startswith('sitemap:'):
-                    sitemap_url = line[8:].strip()
-                    sitemaps.append(sitemap_url)
-            
-            return sitemaps
-            
-        except Exception as e:
-            logger.warning("robots_txt_parsing_failed", base_url=base_url, error=str(e))
-            return []
-    
-    async def _discover_from_search_engines(
-        self, 
-        base_url: str, 
-        options: MapOptions
-    ) -> List[DiscoveredURL]:
-        """Discover URLs using search engines."""
-        try:
-            # Prepare search query
-            parsed_url = urlparse(base_url)
-            domain = parsed_url.netloc
-            
-            if options.search_query and options.allow_external_links:
-                search_query = f"{options.search_query} {domain}"
-            elif options.search_query:
-                search_query = f"{options.search_query} site:{domain}"
-            else:
-                search_query = f"site:{domain}"
-            
-            logger.info("search_engine_discovery_started", 
-                       base_url=base_url, 
-                       query=search_query)
-            
-            # Use multi-search service
-            search_service = await get_multi_search_service()
-            
-            discovered_urls = []
-            total_results_needed = min(options.limit, 1000)  # Cap at 1000
-            
-            # Calculate pages needed
-            results_per_page = options.search_results_per_page
-            pages_needed = min(
-                options.max_search_pages,
-                (total_results_needed + results_per_page - 1) // results_per_page
-            )
-            
-            # Perform searches (for now, single search - can be extended for pagination)
-            search_options = SearchOptions(
-                query=search_query,
-                num_results=total_results_needed,
-                lang="en",
-                country="us"
-            )
-            
-            search_results = await search_service.search(search_options)
-            
-            # Convert search results to discovered URLs
-            for result in search_results:
-                discovered_url = DiscoveredURL(
-                    url=result.url,
-                    source="search",
-                    title=result.title,
-                    description=result.description
-                )
-                discovered_urls.append(discovered_url)
-            
-            logger.info("search_engine_discovery_completed",
-                       base_url=base_url,
-                       urls_discovered=len(discovered_urls))
-            
-            return discovered_urls
-            
-        except Exception as e:
-            logger.error("search_engine_discovery_failed", base_url=base_url, error=str(e))
-            return []
-    
-    async def _discover_from_index(
-        self, 
-        base_url: str, 
-        options: MapOptions
-    ) -> List[DiscoveredURL]:
-        """Discover URLs from internal index/cache."""
-        try:
-            # Check if we have cached results
-            cache_key = f"index:{base_url}"
-            if cache_key in self.discovered_cache:
-                cached_urls = self.discovered_cache[cache_key]
-                logger.info("index_discovery_cache_hit", 
-                           base_url=base_url, 
-                           cached_urls=len(cached_urls))
-                return cached_urls
-            
-            # In a real implementation, this would query an internal URL index
-            # For now, return empty list
-            return []
-            
-        except Exception as e:
-            logger.error("index_discovery_failed", base_url=base_url, error=str(e))
-            return []
-    
-    async def _discover_from_crawling(
-        self, 
-        base_url: str, 
-        options: MapOptions
-    ) -> List[DiscoveredURL]:
-        """Discover URLs through crawling."""
-        try:
-            discovered_urls = []
-            crawled_urls = set()
-            urls_to_crawl = [(base_url, 0)]  # (url, depth)
-            
-            semaphore = asyncio.Semaphore(options.concurrent_requests)
-            
-            while urls_to_crawl and len(discovered_urls) < options.limit:
-                current_batch = []
-                
-                # Prepare batch of URLs to crawl
-                for _ in range(min(options.concurrent_requests, len(urls_to_crawl))):
-                    if urls_to_crawl:
-                        url, depth = urls_to_crawl.pop(0)
-                        if url not in crawled_urls and depth <= options.max_depth:
-                            current_batch.append((url, depth))
-                            crawled_urls.add(url)
-                
-                if not current_batch:
-                    break
-                
-                # Crawl batch
-                async def crawl_url(url_depth_tuple):
-                    async with semaphore:
-                        return await self._crawl_single_url(url_depth_tuple, base_url, options)
-                
-                batch_results = await asyncio.gather(
-                    *[crawl_url(item) for item in current_batch],
-                    return_exceptions=True
-                )
-                
-                # Process results
-                for result in batch_results:
-                    if isinstance(result, dict):
-                        discovered_url = result["discovered_url"]
-                        new_links = result["links"]
-                        
-                        discovered_urls.append(discovered_url)
-                        
-                        # Add new links to crawl queue
-                        for link_url in new_links:
-                            if link_url not in crawled_urls:
-                                urls_to_crawl.append((link_url, discovered_url.depth + 1))
-            
-            logger.info("crawl_discovery_completed",
-                       base_url=base_url,
-                       urls_crawled=len(crawled_urls),
-                       urls_discovered=len(discovered_urls))
-            
-            return discovered_urls[:options.limit]
-            
-        except Exception as e:
-            logger.error("crawl_discovery_failed", base_url=base_url, error=str(e))
-            return []
-    
-    async def _crawl_single_url(
-        self, 
-        url_depth_tuple: tuple, 
-        base_url: str, 
-        options: MapOptions
-    ) -> Dict[str, Any]:
-        """Crawl a single URL and extract links."""
-        url, depth = url_depth_tuple
-        
-        try:
-            response = await self.client.get(url, timeout=options.timeout)
-            
-            discovered_url = DiscoveredURL(
-                url=url,
-                source="crawl",
-                status_code=response.status_code,
-                content_type=response.headers.get("content-type", ""),
-                depth=depth
-            )
-            
-            links = []
-            
-            if response.status_code == 200 and "text/html" in response.headers.get("content-type", ""):
-                # Extract links from HTML
-                from bs4 import BeautifulSoup
-                soup = BeautifulSoup(response.text, 'lxml')
-                
-                # Extract title
-                title_elem = soup.find('title')
-                if title_elem:
-                    discovered_url.title = title_elem.get_text().strip()
-                
-                # Extract meta description
-                meta_desc = soup.find('meta', attrs={'name': 'description'})
-                if meta_desc:
-                    discovered_url.description = meta_desc.get('content', '').strip()
-                
-                # Extract links
-                for link_elem in soup.find_all('a', href=True):
-                    href = link_elem['href']
-                    absolute_url = urljoin(url, href)
-                    
-                    # Filter links based on options
-                    if self._should_include_url(absolute_url, base_url, options):
-                        links.append(absolute_url)
-            
-            return {
-                "discovered_url": discovered_url,
-                "links": links
-            }
-            
-        except Exception as e:
-            logger.warning("url_crawling_failed", url=url, error=str(e))
-            return {
-                "discovered_url": DiscoveredURL(url=url, source="crawl", depth=depth),
-                "links": []
-            }
-    
-    def _should_include_url(self, url: str, base_url: str, options: MapOptions) -> bool:
-        """Check if URL should be included based on options."""
-        try:
-            url_parsed = urlparse(url)
-            base_parsed = urlparse(base_url)
-            
-            # Skip non-HTTP(S) URLs
-            if url_parsed.scheme not in ['http', 'https']:
-                return False
-            
-            # Check domain restrictions
-            if not options.allow_external_links:
-                if options.include_subdomains:
-                    # Allow subdomains
-                    if not url_parsed.netloc.endswith(base_parsed.netloc):
-                        return False
-                else:
-                    # Exact domain match only
-                    if url_parsed.netloc != base_parsed.netloc:
-                        return False
-            
-            # Check path filtering
-            if options.filter_by_path and base_parsed.path and base_parsed.path != "/":
-                if not url_parsed.path.startswith(base_parsed.path):
-                    return False
-            
-            # Check include patterns
-            if options.include_patterns:
-                if not any(re.search(pattern, url) for pattern in options.include_patterns):
-                    return False
-            
-            # Check exclude patterns
-            if options.exclude_patterns:
-                if any(re.search(pattern, url) for pattern in options.exclude_patterns):
-                    return False
-            
-            return True
-            
-        except Exception:
-            return False
-    
-    async def _deduplicate_and_filter(
-        self, 
-        discovered_urls: List[DiscoveredURL], 
-        base_url: str, 
-        options: MapOptions
-    ) -> List[DiscoveredURL]:
-        """Remove duplicates and apply final filtering."""
-        # Deduplicate by URL
-        unique_urls = {}
-        for discovered_url in discovered_urls:
-            url = discovered_url.url
-            
-            # Normalize URL for comparison
-            normalized_url = self._normalize_url(url)
-            
-            if normalized_url not in unique_urls:
-                # Apply final filtering
-                if self._should_include_url(url, base_url, options):
-                    unique_urls[normalized_url] = discovered_url
-            else:
-                # Merge metadata from multiple sources
-                existing = unique_urls[normalized_url]
-                if not existing.title and discovered_url.title:
-                    existing.title = discovered_url.title
-                if not existing.description and discovered_url.description:
-                    existing.description = discovered_url.description
-                if not existing.last_modified and discovered_url.last_modified:
-                    existing.last_modified = discovered_url.last_modified
-        
-        # Sort by source priority and other factors
-        sorted_urls = sorted(
-            unique_urls.values(),
-            key=lambda x: (
-                self._get_source_priority(x.source),
-                -(x.priority or 0),
-                x.url
-            )
-        )
-        
-        return sorted_urls
-    
-    def _normalize_url(self, url: str) -> str:
-        """Normalize URL for comparison."""
-        try:
-            parsed = urlparse(url)
-            
-            # Remove fragment
-            normalized = parsed._replace(fragment="").geturl()
-            
-            # Remove trailing slash for non-root paths
-            if normalized.endswith("/") and parsed.path != "/":
-                normalized = normalized[:-1]
-            
-            # Remove www. for comparison
-            normalized = normalized.replace("://www.", "://")
-            
-            return normalized.lower()
-            
-        except Exception:
-            return url.lower()
-    
-    def _get_source_priority(self, source: str) -> int:
-        """Get priority for source (lower number = higher priority)."""
-        priorities = {
-            "sitemap": 1,
-            "index": 2,
-            "search": 3,
-            "crawl": 4
-        }
-        return priorities.get(source, 5)
-    
-    async def get_mapping_stats(self) -> Dict[str, Any]:
-        """Get mapping service statistics."""
-        return {
-            "mapping_stats": self.mapping_stats,
-            "cache_size": len(self.discovered_cache),
-            "success_rate": (
-                self.mapping_stats["successful_maps"] / 
-                max(1, self.mapping_stats["total_requests"])
-            ) if self.mapping_stats["total_requests"] > 0 else 0,
-            "avg_urls_per_map": (
-                self.mapping_stats["urls_discovered"] /
-                max(1, self.mapping_stats["successful_maps"])
-            ) if self.mapping_stats["successful_maps"] > 0 else 0
-        }
-    
-    async def cleanup(self):
-        """Cleanup resources."""
-        if self.client:
-            await self.client.aclose()
-
-
-# Singleton service
-_website_mapper: Optional[WebsiteMapper] = None
-
-
-async def get_website_mapper() -> WebsiteMapper:
-    """Get or create website mapper service instance."""
-    global _website_mapper
-    
-    if _website_mapper is None:
-        _website_mapper = WebsiteMapper()
-    
-    return _website_mapper
-
-
-# Convenience function
-async def map_website_urls(
-    url: str,
-    strategy: str = "combined",
-    limit: int = 1000,
-    include_subdomains: bool = True,
-    search_query: Optional[str] = None
-) -> WebsiteMapResult:
-    """
-    Convenience function for website mapping.
-    
-    Args:
-        url: Base URL to map
-        strategy: Mapping strategy (sitemap_only, search_engine, combined, crawl_based)
-        limit: Maximum URLs to return
-        include_subdomains: Whether to include subdomains
-        search_query: Optional search query for filtering
-        
-    Returns:
-        WebsiteMapResult with discovered URLs
-    """
-    mapper = await get_website_mapper()
-    
-    options = MapOptions(
-        strategy=MapStrategy(strategy),
-        limit=limit,
-        include_subdomains=include_subdomains,
-        search_query=search_query
-    )
-    
-    return await mapper.map_website(url, options)
diff --git a/apps/backend/app/services/zero_retention.py b/apps/backend/app/services/zero_retention.py
deleted file mode 100644
index 25ac4f6..0000000
--- a/apps/backend/app/services/zero_retention.py
+++ /dev/null
@@ -1,599 +0,0 @@
-"""
-Zero Data Retention service for privacy compliance inspired by Firecrawl.
-
-Provides automatic data deletion capabilities:
-- Automatic deletion after configurable time periods
-- Privacy-compliant data handling
-- Selective retention policies
-- Secure data wiping
-- Compliance tracking and reporting
-"""
-
-import asyncio
-import time
-from typing import Dict, List, Optional, Any, Set
-from dataclasses import dataclass, field
-from datetime import datetime, timedelta
-from enum import Enum
-import structlog
-
-from app.config import get_settings
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-class RetentionPolicy(Enum):
-    """Data retention policies."""
-    ZERO_HOURS = "zero_hours"  # Delete immediately after use
-    ONE_HOUR = "1_hour"
-    SIX_HOURS = "6_hours"
-    TWENTY_FOUR_HOURS = "24_hours"  # Standard zero retention
-    SEVEN_DAYS = "7_days"
-    THIRTY_DAYS = "30_days"
-    PERMANENT = "permanent"  # No auto-deletion
-
-
-class DataType(Enum):
-    """Types of data subject to retention policies."""
-    SCRAPED_CONTENT = "scraped_content"
-    EXTRACTED_DATA = "extracted_data"
-    SEARCH_RESULTS = "search_results"
-    SCREENSHOTS = "screenshots"
-    PDFS = "pdfs"
-    CRAWL_DATA = "crawl_data"
-    BATCH_RESULTS = "batch_results"
-    CHANGE_TRACKING = "change_tracking"
-
-
-@dataclass
-class RetentionRule:
-    """Rule for data retention."""
-    data_type: DataType
-    policy: RetentionPolicy
-    applies_to: List[str] = field(default_factory=list)  # Specific patterns/tags
-    exceptions: List[str] = field(default_factory=list)  # Exclusion patterns
-
-
-@dataclass
-class DataRecord:
-    """Record of data subject to retention policy."""
-    id: str
-    data_type: DataType
-    created_at: datetime
-    expires_at: datetime
-    size_bytes: int = 0
-    metadata: Dict[str, Any] = field(default_factory=dict)
-    tags: List[str] = field(default_factory=list)
-    location: Optional[str] = None  # File path, database key, etc.
-    secure_delete: bool = False
-
-
-@dataclass
-class RetentionReport:
-    """Report on retention policy compliance."""
-    total_records: int
-    expired_records: int
-    deleted_records: int
-    failed_deletions: int
-    bytes_deleted: int
-    compliance_score: float
-    policy_violations: List[str] = field(default_factory=list)
-    generated_at: datetime = field(default_factory=datetime.utcnow)
-
-
-class ZeroRetentionManager:
-    """
-    Zero Data Retention manager for privacy compliance.
-    
-    Provides comprehensive data lifecycle management:
-    - Automatic data expiration and deletion
-    - Flexible retention policies
-    - Secure data wiping
-    - Compliance monitoring and reporting
-    """
-    
-    def __init__(self):
-        """Initialize zero retention manager."""
-        self.data_records: Dict[str, DataRecord] = {}
-        self.retention_rules: List[RetentionRule] = []
-        self.deletion_queue: Set[str] = set()
-        self.stats = {
-            "total_records_managed": 0,
-            "total_deletions": 0,
-            "bytes_deleted": 0,
-            "policy_violations": 0
-        }
-        
-        # Default retention rules
-        self._setup_default_rules()
-        
-        # Start background deletion task
-        self._deletion_task = None
-        asyncio.create_task(self._start_deletion_worker())
-    
-    def _setup_default_rules(self):
-        """Setup default retention rules."""
-        self.retention_rules = [
-            # Zero retention for sensitive data
-            RetentionRule(
-                data_type=DataType.SCRAPED_CONTENT,
-                policy=RetentionPolicy.TWENTY_FOUR_HOURS,
-                applies_to=["zero_retention", "privacy"]
-            ),
-            RetentionRule(
-                data_type=DataType.EXTRACTED_DATA,
-                policy=RetentionPolicy.TWENTY_FOUR_HOURS,
-                applies_to=["zero_retention", "privacy"]
-            ),
-            RetentionRule(
-                data_type=DataType.SCREENSHOTS,
-                policy=RetentionPolicy.ONE_HOUR,
-                applies_to=["temporary", "debug"]
-            ),
-            RetentionRule(
-                data_type=DataType.PDFS,
-                policy=RetentionPolicy.SIX_HOURS,
-                applies_to=["temporary"]
-            ),
-            
-            # Standard retention for operational data
-            RetentionRule(
-                data_type=DataType.SEARCH_RESULTS,
-                policy=RetentionPolicy.SEVEN_DAYS,
-                applies_to=["cache"]
-            ),
-            RetentionRule(
-                data_type=DataType.CHANGE_TRACKING,
-                policy=RetentionPolicy.THIRTY_DAYS,
-                applies_to=["monitoring"]
-            )
-        ]
-    
-    async def register_data(
-        self, 
-        data_id: str,
-        data_type: DataType,
-        size_bytes: int = 0,
-        tags: Optional[List[str]] = None,
-        location: Optional[str] = None,
-        custom_policy: Optional[RetentionPolicy] = None,
-        secure_delete: bool = False,
-        metadata: Optional[Dict[str, Any]] = None
-    ) -> DataRecord:
-        """
-        Register data for retention management.
-        
-        Args:
-            data_id: Unique identifier for the data
-            data_type: Type of data being registered
-            size_bytes: Size of data in bytes
-            tags: Tags for policy matching
-            location: Where the data is stored
-            custom_policy: Override default retention policy
-            secure_delete: Whether to use secure deletion
-            metadata: Additional metadata
-            
-        Returns:
-            DataRecord with retention information
-        """
-        tags = tags or []
-        metadata = metadata or {}
-        
-        # Determine retention policy
-        policy = custom_policy
-        if not policy:
-            policy = self._determine_retention_policy(data_type, tags)
-        
-        # Calculate expiration time
-        expires_at = self._calculate_expiration(policy)
-        
-        # Create data record
-        record = DataRecord(
-            id=data_id,
-            data_type=data_type,
-            created_at=datetime.utcnow(),
-            expires_at=expires_at,
-            size_bytes=size_bytes,
-            metadata=metadata,
-            tags=tags,
-            location=location,
-            secure_delete=secure_delete
-        )
-        
-        self.data_records[data_id] = record
-        self.stats["total_records_managed"] += 1
-        
-        logger.info("data_registered_for_retention",
-                   data_id=data_id,
-                   data_type=data_type.value,
-                   policy=policy.value,
-                   expires_at=expires_at.isoformat(),
-                   size_bytes=size_bytes)
-        
-        return record
-    
-    def _determine_retention_policy(self, data_type: DataType, tags: List[str]) -> RetentionPolicy:
-        """Determine retention policy for data."""
-        # Find matching rule
-        for rule in self.retention_rules:
-            if rule.data_type == data_type:
-                # Check if tags match applies_to criteria
-                if not rule.applies_to or any(tag in rule.applies_to for tag in tags):
-                    # Check exclusions
-                    if not rule.exceptions or not any(tag in rule.exceptions for tag in tags):
-                        return rule.policy
-        
-        # Default to 24 hours if no specific rule found
-        return RetentionPolicy.TWENTY_FOUR_HOURS
-    
-    def _calculate_expiration(self, policy: RetentionPolicy) -> datetime:
-        """Calculate expiration datetime based on policy."""
-        now = datetime.utcnow()
-        
-        if policy == RetentionPolicy.ZERO_HOURS:
-            return now  # Expires immediately
-        elif policy == RetentionPolicy.ONE_HOUR:
-            return now + timedelta(hours=1)
-        elif policy == RetentionPolicy.SIX_HOURS:
-            return now + timedelta(hours=6)
-        elif policy == RetentionPolicy.TWENTY_FOUR_HOURS:
-            return now + timedelta(hours=24)
-        elif policy == RetentionPolicy.SEVEN_DAYS:
-            return now + timedelta(days=7)
-        elif policy == RetentionPolicy.THIRTY_DAYS:
-            return now + timedelta(days=30)
-        else:  # PERMANENT
-            return now + timedelta(days=36500)  # 100 years = effectively permanent
-    
-    async def schedule_deletion(self, data_id: str) -> bool:
-        """Schedule data for deletion."""
-        if data_id not in self.data_records:
-            return False
-        
-        self.deletion_queue.add(data_id)
-        logger.info("data_scheduled_for_deletion", data_id=data_id)
-        return True
-    
-    async def force_delete(self, data_id: str) -> bool:
-        """Force immediate deletion of data."""
-        if data_id not in self.data_records:
-            return False
-        
-        record = self.data_records[data_id]
-        success = await self._delete_data_record(record)
-        
-        if success:
-            del self.data_records[data_id]
-            self.deletion_queue.discard(data_id)
-        
-        return success
-    
-    async def extend_retention(self, data_id: str, additional_hours: int) -> bool:
-        """Extend retention period for data."""
-        if data_id not in self.data_records:
-            return False
-        
-        record = self.data_records[data_id]
-        record.expires_at += timedelta(hours=additional_hours)
-        
-        logger.info("retention_extended",
-                   data_id=data_id,
-                   additional_hours=additional_hours,
-                   new_expires_at=record.expires_at.isoformat())
-        
-        return True
-    
-    async def get_retention_status(self, data_id: str) -> Optional[Dict[str, Any]]:
-        """Get retention status for data."""
-        if data_id not in self.data_records:
-            return None
-        
-        record = self.data_records[data_id]
-        now = datetime.utcnow()
-        
-        return {
-            "data_id": data_id,
-            "data_type": record.data_type.value,
-            "created_at": record.created_at.isoformat(),
-            "expires_at": record.expires_at.isoformat(),
-            "is_expired": now >= record.expires_at,
-            "time_remaining_hours": max(0, (record.expires_at - now).total_seconds() / 3600),
-            "size_bytes": record.size_bytes,
-            "tags": record.tags,
-            "secure_delete": record.secure_delete,
-            "scheduled_for_deletion": data_id in self.deletion_queue
-        }
-    
-    async def generate_compliance_report(self) -> RetentionReport:
-        """Generate compliance report."""
-        now = datetime.utcnow()
-        
-        total_records = len(self.data_records)
-        expired_records = 0
-        policy_violations = []
-        total_size = 0
-        
-        for record in self.data_records.values():
-            total_size += record.size_bytes
-            
-            if now >= record.expires_at:
-                expired_records += 1
-                
-                # Check for policy violations (expired data not deleted)
-                if data_id not in self.deletion_queue:
-                    violation = f"Data {record.id} ({record.data_type.value}) expired at {record.expires_at.isoformat()} but not scheduled for deletion"
-                    policy_violations.append(violation)
-        
-        deleted_records = self.stats["total_deletions"]
-        failed_deletions = expired_records - len(self.deletion_queue)
-        
-        compliance_score = 1.0
-        if total_records > 0:
-            compliance_score = max(0.0, 1.0 - (len(policy_violations) / total_records))
-        
-        return RetentionReport(
-            total_records=total_records,
-            expired_records=expired_records,
-            deleted_records=deleted_records,
-            failed_deletions=max(0, failed_deletions),
-            bytes_deleted=self.stats["bytes_deleted"],
-            compliance_score=compliance_score,
-            policy_violations=policy_violations
-        )
-    
-    async def _start_deletion_worker(self):
-        """Start background worker for data deletion."""
-        self._deletion_task = asyncio.create_task(self._deletion_worker())
-    
-    async def _deletion_worker(self):
-        """Background worker that deletes expired data."""
-        while True:
-            try:
-                await self._process_expired_data()
-                await self._process_deletion_queue()
-                await asyncio.sleep(300)  # Check every 5 minutes
-                
-            except Exception as e:
-                logger.error("deletion_worker_error", error=str(e))
-                await asyncio.sleep(60)  # Wait 1 minute before retrying
-    
-    async def _process_expired_data(self):
-        """Process expired data records."""
-        now = datetime.utcnow()
-        expired_ids = []
-        
-        for data_id, record in self.data_records.items():
-            if now >= record.expires_at and data_id not in self.deletion_queue:
-                expired_ids.append(data_id)
-        
-        for data_id in expired_ids:
-            self.deletion_queue.add(data_id)
-            logger.info("data_expired_scheduled_deletion", data_id=data_id)
-        
-        if expired_ids:
-            logger.info("expired_data_processed", count=len(expired_ids))
-    
-    async def _process_deletion_queue(self):
-        """Process data in deletion queue."""
-        if not self.deletion_queue:
-            return
-        
-        batch_size = 10
-        current_batch = list(self.deletion_queue)[:batch_size]
-        
-        for data_id in current_batch:
-            try:
-                if data_id in self.data_records:
-                    record = self.data_records[data_id]
-                    success = await self._delete_data_record(record)
-                    
-                    if success:
-                        del self.data_records[data_id]
-                        self.deletion_queue.remove(data_id)
-                        self.stats["total_deletions"] += 1
-                        self.stats["bytes_deleted"] += record.size_bytes
-                        
-                        logger.info("data_deleted",
-                                   data_id=data_id,
-                                   data_type=record.data_type.value,
-                                   size_bytes=record.size_bytes)
-                    else:
-                        logger.warning("data_deletion_failed", data_id=data_id)
-                else:
-                    # Record no longer exists, remove from queue
-                    self.deletion_queue.remove(data_id)
-                    
-            except Exception as e:
-                logger.error("deletion_processing_error", 
-                           data_id=data_id, 
-                           error=str(e))
-    
-    async def _delete_data_record(self, record: DataRecord) -> bool:
-        """Delete actual data for a record."""
-        try:
-            # Handle different data storage types
-            if record.location:
-                success = await self._delete_file_data(record)
-            else:
-                success = await self._delete_memory_data(record)
-            
-            # Secure deletion if requested
-            if success and record.secure_delete:
-                await self._secure_wipe(record)
-            
-            return success
-            
-        except Exception as e:
-            logger.error("data_deletion_failed", 
-                        data_id=record.id, 
-                        error=str(e))
-            return False
-    
-    async def _delete_file_data(self, record: DataRecord) -> bool:
-        """Delete file-based data."""
-        try:
-            import os
-            
-            if record.location and os.path.exists(record.location):
-                os.remove(record.location)
-                logger.debug("file_deleted", 
-                           data_id=record.id, 
-                           location=record.location)
-                return True
-            
-            return True  # File doesn't exist, consider it deleted
-            
-        except Exception as e:
-            logger.error("file_deletion_failed", 
-                        data_id=record.id, 
-                        location=record.location,
-                        error=str(e))
-            return False
-    
-    async def _delete_memory_data(self, record: DataRecord) -> bool:
-        """Delete in-memory data."""
-        # This would integrate with your caching/storage systems
-        # For now, we'll just mark it as handled
-        logger.debug("memory_data_deleted", data_id=record.id)
-        return True
-    
-    async def _secure_wipe(self, record: DataRecord):
-        """Perform secure wiping of sensitive data."""
-        try:
-            if record.location:
-                # Secure file wiping (simplified implementation)
-                import os
-                if os.path.exists(record.location):
-                    # Overwrite file with random data multiple times
-                    file_size = os.path.getsize(record.location)
-                    with open(record.location, 'r+b') as f:
-                        for _ in range(3):  # 3-pass overwrite
-                            f.seek(0)
-                            f.write(os.urandom(file_size))
-                            f.flush()
-                            os.fsync(f.fileno())
-                    
-                    os.remove(record.location)
-                    
-            logger.info("secure_wipe_completed", data_id=record.id)
-            
-        except Exception as e:
-            logger.error("secure_wipe_failed", 
-                        data_id=record.id, 
-                        error=str(e))
-    
-    async def get_retention_stats(self) -> Dict[str, Any]:
-        """Get retention service statistics."""
-        now = datetime.utcnow()
-        
-        active_records = len(self.data_records)
-        expired_count = len([r for r in self.data_records.values() if now >= r.expires_at])
-        queue_size = len(self.deletion_queue)
-        
-        total_size = sum(r.size_bytes for r in self.data_records.values())
-        
-        return {
-            "retention_stats": self.stats,
-            "active_records": active_records,
-            "expired_records": expired_count,
-            "deletion_queue_size": queue_size,
-            "total_data_size_bytes": total_size,
-            "policies_count": len(self.retention_rules),
-            "worker_running": self._deletion_task is not None and not self._deletion_task.done()
-        }
-    
-    def add_retention_rule(self, rule: RetentionRule):
-        """Add custom retention rule."""
-        self.retention_rules.append(rule)
-        logger.info("retention_rule_added", 
-                   data_type=rule.data_type.value,
-                   policy=rule.policy.value)
-    
-    def remove_retention_rule(self, data_type: DataType, policy: RetentionPolicy) -> bool:
-        """Remove retention rule."""
-        for i, rule in enumerate(self.retention_rules):
-            if rule.data_type == data_type and rule.policy == policy:
-                del self.retention_rules[i]
-                logger.info("retention_rule_removed", 
-                           data_type=data_type.value,
-                           policy=policy.value)
-                return True
-        return False
-    
-    async def cleanup(self):
-        """Cleanup resources."""
-        if self._deletion_task:
-            self._deletion_task.cancel()
-            try:
-                await self._deletion_task
-            except asyncio.CancelledError:
-                pass
-
-
-# Singleton service
-_zero_retention_manager: Optional[ZeroRetentionManager] = None
-
-
-async def get_zero_retention_manager() -> ZeroRetentionManager:
-    """Get or create zero retention manager instance."""
-    global _zero_retention_manager
-    
-    if _zero_retention_manager is None:
-        _zero_retention_manager = ZeroRetentionManager()
-    
-    return _zero_retention_manager
-
-
-# Convenience functions
-async def register_for_zero_retention(
-    data_id: str,
-    data_type: str,
-    size_bytes: int = 0,
-    location: Optional[str] = None
-) -> bool:
-    """
-    Register data for zero retention (24-hour deletion).
-    
-    Args:
-        data_id: Unique data identifier
-        data_type: Type of data
-        size_bytes: Size in bytes
-        location: Storage location
-        
-    Returns:
-        True if registered successfully
-    """
-    manager = await get_zero_retention_manager()
-    
-    try:
-        data_type_enum = DataType(data_type)
-    except ValueError:
-        data_type_enum = DataType.SCRAPED_CONTENT
-    
-    record = await manager.register_data(
-        data_id=data_id,
-        data_type=data_type_enum,
-        size_bytes=size_bytes,
-        location=location,
-        tags=["zero_retention"],
-        secure_delete=True
-    )
-    
-    return record is not None
-
-
-async def check_compliance_status() -> Dict[str, Any]:
-    """Check overall compliance status."""
-    manager = await get_zero_retention_manager()
-    report = await manager.generate_compliance_report()
-    
-    return {
-        "compliant": report.compliance_score >= 0.95,
-        "compliance_score": report.compliance_score,
-        "total_records": report.total_records,
-        "expired_records": report.expired_records,
-        "policy_violations": len(report.policy_violations),
-        "bytes_deleted": report.bytes_deleted,
-        "report_time": report.generated_at.isoformat()
-    }
diff --git a/apps/backend/app/utils/__init__.py b/apps/backend/app/utils/__init__.py
deleted file mode 100644
index d62c05a..0000000
--- a/apps/backend/app/utils/__init__.py
+++ /dev/null
@@ -1,92 +0,0 @@
-"""
-Utility functions for the UnSearch API.
-"""
-from app.utils.text_processing import (
-    sanitize_text,
-    extract_snippet,
-    detect_language,
-    calculate_text_quality,
-    extract_keywords,
-    truncate_text,
-    normalize_url
-)
-from app.utils.validators import (
-    validate_query,
-    validate_url,
-    validate_engines,
-    validate_language_code,
-    validate_css_selector,
-    validate_custom_selectors,
-    validate_webhook_url,
-    validate_timeout,
-    validate_cache_ttl,
-    validate_max_results
-)
-from app.utils.security import (
-    generate_api_key,
-    hash_password,
-    verify_password,
-    sanitize_input,
-    is_safe_url,
-    generate_csrf_token,
-    verify_csrf_token,
-    SecurityHeaders
-)
-from app.utils.exceptions import (
-    UnQuestException,
-    SearXNGException,
-    ScrapingException,
-    CacheException,
-    DatabaseException,
-    BadRequestException,
-    UnauthorizedException,
-    ForbiddenException,
-    NotFoundException,
-    TooManyRequestsException,
-    InternalServerErrorException,
-    ServiceUnavailableException
-)
-
-__all__ = [
-    # Text processing
-    "sanitize_text",
-    "extract_snippet",
-    "detect_language",
-    "calculate_text_quality",
-    "extract_keywords",
-    "truncate_text",
-    "normalize_url",
-    # Validators
-    "validate_query",
-    "validate_url",
-    "validate_engines",
-    "validate_language_code",
-    "validate_css_selector",
-    "validate_custom_selectors",
-    "validate_webhook_url",
-    "validate_timeout",
-    "validate_cache_ttl",
-    "validate_max_results",
-    # Security
-    "generate_api_key",
-    "hash_password",
-    "verify_password",
-    "sanitize_input",
-    "is_safe_url",
-    "generate_csrf_token",
-    "verify_csrf_token",
-    "SecurityHeaders",
-    # Exceptions
-    "UnQuestException",
-    "SearXNGException",
-    "ScrapingException",
-    "CacheException",
-    "DatabaseException",
-    "BadRequestException",
-    "UnauthorizedException",
-    "ForbiddenException",
-    "NotFoundException",
-    "TooManyRequestsException",
-    "InternalServerErrorException",
-    "ServiceUnavailableException",
-]
diff --git a/apps/backend/app/utils/error_handlers.py b/apps/backend/app/utils/error_handlers.py
deleted file mode 100644
index 3aee781..0000000
--- a/apps/backend/app/utils/error_handlers.py
+++ /dev/null
@@ -1,302 +0,0 @@
-"""
-Global error handlers for the UnQuest API.
-"""
-import traceback
-from typing import Union
-from fastapi import Request, HTTPException, status
-from fastapi.responses import JSONResponse
-from fastapi.exceptions import RequestValidationError
-from slowapi.errors import RateLimitExceeded
-from httpx import HTTPError, TimeoutException
-import structlog
-
-from app.config import get_settings
-from app.models.responses import ErrorResponse
-from app.utils.exceptions import (
-    UnQuestException, SearXNGException, ScrapingException,
-    CacheException, DatabaseException, UnQuestHTTPException
-)
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-
-async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
-    """Handle request validation errors."""
-    logger.warning(
-        "validation_error",
-        path=request.url.path,
-        method=request.method,
-        errors=exc.errors(),
-        client_ip=request.client.host if request.client else None
-    )
-    
-    # Format validation errors
-    formatted_errors = []
-    for error in exc.errors():
-        field = ".".join(str(loc) for loc in error["loc"])
-        formatted_errors.append({
-            "field": field,
-            "message": error["msg"],
-            "type": error["type"]
-        })
-    
-    error_response = ErrorResponse(
-        error="ValidationError",
-        message="Request validation failed",
-        details={"validation_errors": formatted_errors},
-        request_id=request.headers.get("X-Request-ID")
-    )
-    
-    return JSONResponse(
-        status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-        content=error_response.dict()
-    )
-
-
-async def rate_limit_exception_handler(request: Request, exc: RateLimitExceeded) -> JSONResponse:
-    """Handle rate limiting errors."""
-    logger.warning(
-        "rate_limit_exceeded",
-        path=request.url.path,
-        method=request.method,
-        client_ip=request.client.host if request.client else None,
-        rate_limit=str(exc)
-    )
-    
-    error_response = ErrorResponse(
-        error="RateLimitExceeded",
-        message="Rate limit exceeded. Please try again later.",
-        details={
-            "limit": str(exc),
-            "retry_after": getattr(exc, "retry_after", 60)
-        },
-        request_id=request.headers.get("X-Request-ID")
-    )
-    
-    return JSONResponse(
-        status_code=status.HTTP_429_TOO_MANY_REQUESTS,
-        content=error_response.dict(),
-        headers={"Retry-After": str(getattr(exc, "retry_after", 60))}
-    )
-
-
-async def searxng_exception_handler(request: Request, exc: SearXNGException) -> JSONResponse:
-    """Handle SearXNG service errors."""
-    logger.error(
-        "searxng_error",
-        path=request.url.path,
-        error=str(exc),
-        details=exc.details,
-        client_ip=request.client.host if request.client else None
-    )
-    
-    error_response = ErrorResponse(
-        error="SearXNGError",
-        message="Search service temporarily unavailable",
-        details=exc.details if settings.debug else None,
-        request_id=request.headers.get("X-Request-ID")
-    )
-    
-    return JSONResponse(
-        status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-        content=error_response.dict(),
-        headers={"Retry-After": "60"}
-    )
-
-
-async def scraping_exception_handler(request: Request, exc: ScrapingException) -> JSONResponse:
-    """Handle content scraping errors."""
-    logger.error(
-        "scraping_error",
-        path=request.url.path,
-        error=str(exc),
-        details=exc.details,
-        client_ip=request.client.host if request.client else None
-    )
-    
-    error_response = ErrorResponse(
-        error="ScrapingError",
-        message="Content scraping failed",
-        details=exc.details if settings.debug else None,
-        request_id=request.headers.get("X-Request-ID")
-    )
-    
-    return JSONResponse(
-        status_code=status.HTTP_502_BAD_GATEWAY,
-        content=error_response.dict()
-    )
-
-
-async def cache_exception_handler(request: Request, exc: CacheException) -> JSONResponse:
-    """Handle cache service errors."""
-    logger.error(
-        "cache_error",
-        path=request.url.path,
-        error=str(exc),
-        details=exc.details,
-        client_ip=request.client.host if request.client else None
-    )
-    
-    # Cache errors are usually non-critical, continue without cache
-    error_response = ErrorResponse(
-        error="CacheError",
-        message="Cache service temporarily unavailable",
-        details={"note": "Request processed without caching"} if settings.debug else None,
-        request_id=request.headers.get("X-Request-ID")
-    )
-    
-    return JSONResponse(
-        status_code=status.HTTP_200_OK,  # Continue processing
-        content=error_response.dict()
-    )
-
-
-async def database_exception_handler(request: Request, exc: DatabaseException) -> JSONResponse:
-    """Handle database errors."""
-    logger.error(
-        "database_error",
-        path=request.url.path,
-        error=str(exc),
-        details=exc.details,
-        client_ip=request.client.host if request.client else None
-    )
-    
-    error_response = ErrorResponse(
-        error="DatabaseError",
-        message="Database service temporarily unavailable",
-        details=exc.details if settings.debug else None,
-        request_id=request.headers.get("X-Request-ID")
-    )
-    
-    return JSONResponse(
-        status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-        content=error_response.dict(),
-        headers={"Retry-After": "30"}
-    )
-
-
-async def http_error_handler(request: Request, exc: HTTPError) -> JSONResponse:
-    """Handle HTTP client errors (httpx)."""
-    logger.error(
-        "http_client_error",
-        path=request.url.path,
-        error=str(exc),
-        error_type=type(exc).__name__,
-        client_ip=request.client.host if request.client else None
-    )
-    
-    if isinstance(exc, TimeoutException):
-        error_code = "RequestTimeout"
-        message = "Request timed out"
-        status_code = status.HTTP_504_GATEWAY_TIMEOUT
-    else:
-        error_code = "HTTPError"
-        message = "External service error"
-        status_code = status.HTTP_502_BAD_GATEWAY
-    
-    error_response = ErrorResponse(
-        error=error_code,
-        message=message,
-        details={"error": str(exc)} if settings.debug else None,
-        request_id=request.headers.get("X-Request-ID")
-    )
-    
-    return JSONResponse(
-        status_code=status_code,
-        content=error_response.dict()
-    )
-
-
-async def UnQuest_http_exception_handler(request: Request, exc: UnQuestHTTPException) -> JSONResponse:
-    """Handle custom UnQuest HTTP exceptions."""
-    logger.warning(
-        "UnQuest_http_error",
-        path=request.url.path,
-        error_code=exc.error_code,
-        message=exc.message,
-        status_code=exc.status_code,
-        client_ip=request.client.host if request.client else None
-    )
-    
-    error_response = ErrorResponse(
-        error=exc.error_code,
-        message=exc.message,
-        details=exc.details,
-        request_id=request.headers.get("X-Request-ID")
-    )
-    
-    return JSONResponse(
-        status_code=exc.status_code,
-        content=error_response.dict(),
-        headers=exc.headers
-    )
-
-
-async def generic_exception_handler(request: Request, exc: Exception) -> JSONResponse:
-    """Handle all other unhandled exceptions."""
-    request_id = request.headers.get("X-Request-ID", "unknown")
-    
-    logger.error(
-        "unhandled_exception",
-        path=request.url.path,
-        method=request.method,
-        error=str(exc),
-        error_type=type(exc).__name__,
-        request_id=request_id,
-        client_ip=request.client.host if request.client else None,
-        stack_trace=traceback.format_exc() if settings.debug else None
-    )
-    
-    # Log to database if possible
-    try:
-        from app.services.database import get_database_service
-        db = await get_database_service()
-        await db.log_error(
-            error_type=type(exc).__name__,
-            error_message=str(exc),
-            request_id=request_id,
-            stack_trace=traceback.format_exc(),
-            endpoint=request.url.path,
-            method=request.method,
-            client_ip=request.client.host if request.client else None
-        )
-    except:
-        # Don't fail if database logging fails
-        pass
-    
-    error_response = ErrorResponse(
-        error="InternalServerError",
-        message="An unexpected error occurred",
-        details={
-            "error": str(exc),
-            "type": type(exc).__name__
-        } if settings.debug else None,
-        request_id=request_id
-    )
-    
-    return JSONResponse(
-        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-        content=error_response.dict()
-    )
-
-
-# Exception handler mapping
-EXCEPTION_HANDLERS = {
-    RequestValidationError: validation_exception_handler,
-    RateLimitExceeded: rate_limit_exception_handler,
-    SearXNGException: searxng_exception_handler,
-    ScrapingException: scraping_exception_handler,
-    CacheException: cache_exception_handler,
-    DatabaseException: database_exception_handler,
-    HTTPError: http_error_handler,
-    TimeoutException: http_error_handler,
-    UnQuestHTTPException: UnQuest_http_exception_handler,
-    Exception: generic_exception_handler,
-}
-
-
-def register_exception_handlers(app):
-    """Register all exception handlers with the FastAPI app."""
-    for exception_class, handler in EXCEPTION_HANDLERS.items():
-        app.add_exception_handler(exception_class, handler)
diff --git a/apps/backend/app/utils/exceptions.py b/apps/backend/app/utils/exceptions.py
deleted file mode 100644
index df4ccb2..0000000
--- a/apps/backend/app/utils/exceptions.py
+++ /dev/null
@@ -1,190 +0,0 @@
-"""
-Custom exceptions for the UnQuest API.
-"""
-from typing import Optional, Dict, Any
-from fastapi import HTTPException, status
-
-
-class UnQuestException(Exception):
-    """Base exception for UnQuest API."""
-    
-    def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
-        self.message = message
-        self.details = details or {}
-        super().__init__(self.message)
-
-
-class SearXNGException(UnQuestException):
-    """Exception related to SearXNG operations."""
-    pass
-
-
-class ScrapingException(UnQuestException):
-    """Exception related to content scraping."""
-    pass
-
-
-class CacheException(UnQuestException):
-    """Exception related to cache operations."""
-    pass
-
-
-class DatabaseException(UnQuestException):
-    """Exception related to database operations."""
-    pass
-
-
-class RateLimitException(UnQuestException):
-    """Exception for rate limiting violations."""
-    pass
-
-
-class AuthenticationException(UnQuestException):
-    """Exception for authentication failures."""
-    pass
-
-
-class ValidationException(UnQuestException):
-    """Exception for request validation failures."""
-    pass
-
-
-# HTTP Exception classes with proper status codes
-class UnQuestHTTPException(HTTPException):
-    """Base HTTP exception with structured error response."""
-    
-    def __init__(
-        self,
-        status_code: int,
-        error_code: str,
-        message: str,
-        details: Optional[Dict[str, Any]] = None,
-        headers: Optional[Dict[str, str]] = None
-    ):
-        self.error_code = error_code
-        self.details = details or {}
-        
-        detail = {
-            "error": error_code,
-            "message": message,
-            "details": self.details
-        }
-        
-        super().__init__(status_code=status_code, detail=detail, headers=headers)
-
-
-class BadRequestException(UnQuestHTTPException):
-    """400 Bad Request."""
-    
-    def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
-        super().__init__(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            error_code="BadRequest",
-            message=message,
-            details=details
-        )
-
-
-class UnauthorizedException(UnQuestHTTPException):
-    """401 Unauthorized."""
-    
-    def __init__(self, message: str = "Authentication required", details: Optional[Dict[str, Any]] = None):
-        super().__init__(
-            status_code=status.HTTP_401_UNAUTHORIZED,
-            error_code="Unauthorized",
-            message=message,
-            details=details,
-            headers={"WWW-Authenticate": "Bearer"}
-        )
-
-
-class ForbiddenException(UnQuestHTTPException):
-    """403 Forbidden."""
-    
-    def __init__(self, message: str = "Access forbidden", details: Optional[Dict[str, Any]] = None):
-        super().__init__(
-            status_code=status.HTTP_403_FORBIDDEN,
-            error_code="Forbidden",
-            message=message,
-            details=details
-        )
-
-
-class NotFoundException(UnQuestHTTPException):
-    """404 Not Found."""
-    
-    def __init__(self, message: str = "Resource not found", details: Optional[Dict[str, Any]] = None):
-        super().__init__(
-            status_code=status.HTTP_404_NOT_FOUND,
-            error_code="NotFound",
-            message=message,
-            details=details
-        )
-
-
-class TooManyRequestsException(UnQuestHTTPException):
-    """429 Too Many Requests."""
-    
-    def __init__(
-        self, 
-        message: str = "Rate limit exceeded", 
-        details: Optional[Dict[str, Any]] = None,
-        retry_after: Optional[int] = None
-    ):
-        headers = {}
-        if retry_after:
-            headers["Retry-After"] = str(retry_after)
-            
-        super().__init__(
-            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
-            error_code="TooManyRequests",
-            message=message,
-            details=details,
-            headers=headers
-        )
-
-
-class InternalServerErrorException(UnQuestHTTPException):
-    """500 Internal Server Error."""
-    
-    def __init__(self, message: str = "Internal server error", details: Optional[Dict[str, Any]] = None):
-        super().__init__(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            error_code="InternalServerError",
-            message=message,
-            details=details
-        )
-
-
-class ServiceUnavailableException(UnQuestHTTPException):
-    """503 Service Unavailable."""
-    
-    def __init__(
-        self, 
-        message: str = "Service temporarily unavailable", 
-        details: Optional[Dict[str, Any]] = None,
-        retry_after: Optional[int] = None
-    ):
-        headers = {}
-        if retry_after:
-            headers["Retry-After"] = str(retry_after)
-            
-        super().__init__(
-            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-            error_code="ServiceUnavailable",
-            message=message,
-            details=details,
-            headers=headers
-        )
-
-
-class GatewayTimeoutException(UnQuestHTTPException):
-    """504 Gateway Timeout."""
-    
-    def __init__(self, message: str = "Request timeout", details: Optional[Dict[str, Any]] = None):
-        super().__init__(
-            status_code=status.HTTP_504_GATEWAY_TIMEOUT,
-            error_code="GatewayTimeout",
-            message=message,
-            details=details
-        )
diff --git a/apps/backend/app/utils/security.py b/apps/backend/app/utils/security.py
deleted file mode 100644
index 819b226..0000000
--- a/apps/backend/app/utils/security.py
+++ /dev/null
@@ -1,308 +0,0 @@
-"""
-Security utilities for the UnSearch API.
-"""
-import hashlib
-import hmac
-import secrets
-import time
-from typing import Optional, Dict, Any
-import re
-from urllib.parse import urlparse, urljoin
-import ipaddress
-from fastapi import Request
-
-
-def generate_api_key(length: int = 32) -> str:
-    """
-    Generate a secure API key.
-    
-    Args:
-        length: Length of the API key
-        
-    Returns:
-        Generated API key
-    """
-    return secrets.token_urlsafe(length)
-
-
-def hash_password(password: str, salt: Optional[str] = None) -> tuple[str, str]:
-    """
-    Hash a password with salt.
-    
-    Args:
-        password: Password to hash
-        salt: Optional salt (generated if not provided)
-        
-    Returns:
-        Tuple of (hashed_password, salt)
-    """
-    if salt is None:
-        salt = secrets.token_hex(16)
-    
-    # Use PBKDF2 with SHA-256
-    hashed = hashlib.pbkdf2_hmac('sha256', password.encode(), salt.encode(), 100000)
-    return hashed.hex(), salt
-
-
-def verify_password(password: str, hashed_password: str, salt: str) -> bool:
-    """
-    Verify a password against its hash.
-    
-    Args:
-        password: Password to verify
-        hashed_password: Hashed password
-        salt: Salt used for hashing
-        
-    Returns:
-        True if password is correct
-    """
-    computed_hash, _ = hash_password(password, salt)
-    return hmac.compare_digest(computed_hash, hashed_password)
-
-
-def sanitize_input(text: str, max_length: int = 1000) -> str:
-    """
-    Sanitize user input to prevent XSS and injection attacks.
-    
-    Args:
-        text: Input text to sanitize
-        max_length: Maximum allowed length
-        
-    Returns:
-        Sanitized text
-    """
-    if not text:
-        return ""
-    
-    # Truncate to max length
-    text = text[:max_length]
-    
-    # Remove potentially dangerous characters
-    text = re.sub(r'[<>"\']', '', text)
-    
-    # Remove SQL injection patterns
-    sql_patterns = [
-        r'(\bUNION\b)', r'(\bSELECT\b)', r'(\bINSERT\b)', r'(\bUPDATE\b)',
-        r'(\bDELETE\b)', r'(\bDROP\b)', r'(\bCREATE\b)', r'(\bALTER\b)',
-        r'(\bEXEC\b)', r'(\bEXECUTE\b)', r'(--)', r'(/\*)', r'(\*/)', 
-        r'(\bSCRIPT\b)', r'(\bJAVASCRIPT\b)', r'(\bVBSCRIPT\b)'
-    ]
-    
-    for pattern in sql_patterns:
-        text = re.sub(pattern, '', text, flags=re.IGNORECASE)
-    
-    return text.strip()
-
-
-def is_safe_url(url: str, allowed_hosts: Optional[list] = None) -> bool:
-    """
-    Check if a URL is safe for redirection or webhooks.
-    
-    Args:
-        url: URL to validate
-        allowed_hosts: Optional list of allowed hostnames
-        
-    Returns:
-        True if URL is safe
-    """
-    try:
-        parsed = urlparse(url)
-        
-        # Must have scheme and netloc
-        if not parsed.scheme or not parsed.netloc:
-            return False
-        
-        # Only allow HTTP/HTTPS
-        if parsed.scheme not in ['http', 'https']:
-            return False
-        
-        # Check for localhost/private IPs
-        hostname = parsed.hostname
-        if hostname:
-            try:
-                ip = ipaddress.ip_address(hostname)
-                if ip.is_private or ip.is_loopback or ip.is_reserved:
-                    return False
-            except ValueError:
-                # Not an IP address, check hostname
-                if hostname.lower() in ['localhost', '127.0.0.1', '::1']:
-                    return False
-        
-        # Check allowed hosts if specified
-        if allowed_hosts and hostname not in allowed_hosts:
-            return False
-            
-        return True
-        
-    except Exception:
-        return False
-
-
-def generate_csrf_token() -> str:
-    """Generate a CSRF token."""
-    return secrets.token_urlsafe(32)
-
-
-def verify_csrf_token(token: str, expected_token: str) -> bool:
-    """Verify a CSRF token."""
-    return hmac.compare_digest(token, expected_token)
-
-
-def rate_limit_key(request: Request, identifier: Optional[str] = None) -> str:
-    """
-    Generate a rate limiting key for a request.
-    
-    Args:
-        request: FastAPI request object
-        identifier: Optional identifier (API key, user ID, etc.)
-        
-    Returns:
-        Rate limiting key
-    """
-    if identifier:
-        return f"rate_limit:{identifier}"
-    
-    # Use client IP as fallback
-    client_ip = request.client.host if request.client else "unknown"
-    return f"rate_limit:ip:{client_ip}"
-
-
-def mask_sensitive_data(data: Dict[str, Any], sensitive_keys: Optional[list] = None) -> Dict[str, Any]:
-    """
-    Mask sensitive data in a dictionary for logging.
-    
-    Args:
-        data: Dictionary to mask
-        sensitive_keys: List of keys to mask
-        
-    Returns:
-        Dictionary with masked sensitive data
-    """
-    if sensitive_keys is None:
-        sensitive_keys = [
-            'password', 'api_key', 'token', 'secret', 'auth', 'authorization',
-            'x-api-key', 'cookie', 'session', 'private', 'credential'
-        ]
-    
-    masked_data = {}
-    
-    for key, value in data.items():
-        key_lower = key.lower()
-        
-        if any(sensitive in key_lower for sensitive in sensitive_keys):
-            if isinstance(value, str) and len(value) > 8:
-                masked_data[key] = value[:4] + "****" + value[-4:]
-            else:
-                masked_data[key] = "****"
-        elif isinstance(value, dict):
-            masked_data[key] = mask_sensitive_data(value, sensitive_keys)
-        else:
-            masked_data[key] = value
-    
-    return masked_data
-
-
-def generate_request_id() -> str:
-    """Generate a unique request ID."""
-    timestamp = str(int(time.time() * 1000))
-    random_part = secrets.token_hex(8)
-    return f"req_{timestamp}_{random_part}"
-
-
-def validate_webhook_signature(payload: bytes, signature: str, secret: str) -> bool:
-    """
-    Validate webhook signature using HMAC-SHA256.
-    
-    Args:
-        payload: Raw webhook payload
-        signature: Signature to verify
-        secret: Secret key
-        
-    Returns:
-        True if signature is valid
-    """
-    expected_signature = hmac.new(
-        secret.encode(),
-        payload,
-        hashlib.sha256
-    ).hexdigest()
-    
-    return hmac.compare_digest(f"sha256={expected_signature}", signature)
-
-
-def extract_client_info(request: Request) -> Dict[str, Any]:
-    """
-    Extract client information from request for security logging.
-    
-    Args:
-        request: FastAPI request object
-        
-    Returns:
-        Dictionary with client information
-    """
-    headers = dict(request.headers)
-    
-    return {
-        "ip": request.client.host if request.client else None,
-        "user_agent": headers.get("user-agent"),
-        "referer": headers.get("referer"),
-        "origin": headers.get("origin"),
-        "x_forwarded_for": headers.get("x-forwarded-for"),
-        "x_real_ip": headers.get("x-real-ip"),
-        "cf_connecting_ip": headers.get("cf-connecting-ip"),  # Cloudflare
-        "x_forwarded_proto": headers.get("x-forwarded-proto"),
-        "accept_language": headers.get("accept-language"),
-        "accept_encoding": headers.get("accept-encoding"),
-    }
-
-
-def is_suspicious_request(request: Request) -> bool:
-    """
-    Check if a request looks suspicious.
-    
-    Args:
-        request: FastAPI request object
-        
-    Returns:
-        True if request is suspicious
-    """
-    user_agent = request.headers.get("user-agent", "").lower()
-    
-    # Common bot/scanner signatures
-    suspicious_agents = [
-        'sqlmap', 'nikto', 'nmap', 'masscan', 'zap', 'burp',
-        'acunetix', 'nessus', 'openvas', 'w3af', 'dirb',
-        'gobuster', 'dirbuster', 'wfuzz', 'ffuf'
-    ]
-    
-    if any(agent in user_agent for agent in suspicious_agents):
-        return True
-    
-    # Check for suspicious paths
-    path = request.url.path.lower()
-    suspicious_paths = [
-        'admin', 'phpmyadmin', 'wp-admin', 'config', 'backup',
-        'shell', 'cmd', 'eval', 'exec', 'system', 'passwd'
-    ]
-    
-    if any(path in suspicious_paths for path in suspicious_paths):
-        return True
-    
-    return False
-
-
-class SecurityHeaders:
-    """Security headers for HTTP responses."""
-    
-    @staticmethod
-    def get_headers() -> Dict[str, str]:
-        """Get security headers."""
-        return {
-            "X-Content-Type-Options": "nosniff",
-            "X-Frame-Options": "DENY",
-            "X-XSS-Protection": "1; mode=block",
-            "Strict-Transport-Security": "max-age=31536000; includeSubDomains",
-            "Referrer-Policy": "strict-origin-when-cross-origin",
-            "Permissions-Policy": "geolocation=(), microphone=(), camera=()",
-            "Content-Security-Policy": "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'",
-        }
diff --git a/apps/backend/app/utils/text_processing.py b/apps/backend/app/utils/text_processing.py
deleted file mode 100644
index 44c8eb6..0000000
--- a/apps/backend/app/utils/text_processing.py
+++ /dev/null
@@ -1,321 +0,0 @@
-"""
-Text processing utilities for content extraction and sanitization.
-"""
-import re
-import html
-from typing import List, Optional
-import unicodedata
-from langdetect import detect, LangDetectException
-import nltk
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize, sent_tokenize
-
-# Download required NLTK data
-try:
-    nltk.data.find('tokenizers/punkt')
-except LookupError:
-    nltk.download('punkt', quiet=True)
-
-try:
-    nltk.data.find('corpora/stopwords')
-except LookupError:
-    nltk.download('stopwords', quiet=True)
-
-
-def sanitize_text(text: str) -> str:
-    """
-    Sanitize text by removing HTML entities, extra whitespace, and control characters.
-    
-    Args:
-        text: Raw text to sanitize
-        
-    Returns:
-        Cleaned text
-    """
-    if not text:
-        return ""
-        
-    # Decode HTML entities
-    text = html.unescape(text)
-    
-    # Remove HTML tags
-    text = re.sub(r'<[^>]+>', '', text)
-    
-    # Remove control characters except newlines and tabs
-    text = ''.join(char for char in text if char == '\n' or char == '\t' or not unicodedata.category(char).startswith('C'))
-    
-    # Normalize whitespace
-    text = re.sub(r'\s+', ' ', text)
-    
-    # Remove leading/trailing whitespace
-    text = text.strip()
-    
-    return text
-
-
-def extract_snippet(text: str, query: str, max_length: int = 200, context_words: int = 10) -> str:
-    """
-    Extract a relevant snippet from text based on query terms.
-    
-    Args:
-        text: Full text to extract from
-        query: Search query
-        max_length: Maximum snippet length
-        context_words: Number of words to include before/after match
-        
-    Returns:
-        Extracted snippet with ellipsis if truncated
-    """
-    if not text:
-        return ""
-        
-    # Clean text
-    text = sanitize_text(text)
-    
-    if len(text) <= max_length:
-        return text
-        
-    # Extract query terms
-    query_terms = [term.lower() for term in query.split() if len(term) > 2]
-    
-    if not query_terms:
-        # If no valid query terms, return beginning of text
-        return text[:max_length] + "..." if len(text) > max_length else text
-        
-    # Find best matching sentence
-    sentences = sent_tokenize(text)
-    best_sentence = None
-    best_score = 0
-    
-    for sentence in sentences:
-        sentence_lower = sentence.lower()
-        score = sum(1 for term in query_terms if term in sentence_lower)
-        
-        if score > best_score:
-            best_score = score
-            best_sentence = sentence
-            
-    if best_sentence and len(best_sentence) <= max_length:
-        return best_sentence
-        
-    # Find first occurrence of any query term
-    text_lower = text.lower()
-    first_match_pos = len(text)
-    
-    for term in query_terms:
-        pos = text_lower.find(term)
-        if pos != -1 and pos < first_match_pos:
-            first_match_pos = pos
-            
-    if first_match_pos == len(text):
-        # No match found, return beginning
-        return text[:max_length] + "..."
-        
-    # Extract context around match
-    words = text.split()
-    word_positions = []
-    current_pos = 0
-    
-    for word in words:
-        word_positions.append((current_pos, current_pos + len(word)))
-        current_pos += len(word) + 1  # +1 for space
-        
-    # Find word containing match
-    match_word_idx = 0
-    for idx, (start, end) in enumerate(word_positions):
-        if start <= first_match_pos <= end:
-            match_word_idx = idx
-            break
-            
-    # Extract snippet with context
-    start_idx = max(0, match_word_idx - context_words)
-    end_idx = min(len(words), match_word_idx + context_words + 1)
-    
-    snippet_words = words[start_idx:end_idx]
-    snippet = ' '.join(snippet_words)
-    
-    # Add ellipsis
-    if start_idx > 0:
-        snippet = "..." + snippet
-    if end_idx < len(words):
-        snippet = snippet + "..."
-        
-    # Ensure snippet isn't too long
-    if len(snippet) > max_length:
-        snippet = snippet[:max_length-3] + "..."
-        
-    return snippet
-
-
-def detect_language(text: str) -> Optional[str]:
-    """
-    Detect the language of text.
-    
-    Args:
-        text: Text to analyze
-        
-    Returns:
-        ISO 639-1 language code or None if detection fails
-    """
-    if not text or len(text) < 20:
-        return None
-        
-    try:
-        return detect(text)
-    except LangDetectException:
-        return None
-
-
-def calculate_text_quality(text: str, min_words: int = 50) -> float:
-    """
-    Calculate quality score for extracted text.
-    
-    Args:
-        text: Text to analyze
-        min_words: Minimum words for good quality
-        
-    Returns:
-        Quality score between 0.0 and 1.0
-    """
-    if not text:
-        return 0.0
-        
-    # Clean text
-    text = sanitize_text(text)
-    
-    # Basic metrics
-    words = text.split()
-    word_count = len(words)
-    
-    if word_count < min_words:
-        return word_count / min_words * 0.5
-        
-    # Calculate various quality indicators
-    scores = []
-    
-    # Word count score
-    word_score = min(1.0, word_count / 500)  # Normalize to 500 words
-    scores.append(word_score)
-    
-    # Sentence structure score
-    sentences = sent_tokenize(text)
-    if sentences:
-        avg_sentence_length = word_count / len(sentences)
-        # Optimal sentence length is 15-20 words
-        if 15 <= avg_sentence_length <= 20:
-            sentence_score = 1.0
-        elif 10 <= avg_sentence_length <= 30:
-            sentence_score = 0.8
-        else:
-            sentence_score = 0.5
-        scores.append(sentence_score)
-    
-    # Vocabulary diversity score
-    unique_words = set(word.lower() for word in words if len(word) > 3)
-    diversity_score = min(1.0, len(unique_words) / (word_count * 0.5))
-    scores.append(diversity_score)
-    
-    # Alphanumeric ratio (detect gibberish)
-    alphanumeric_chars = sum(1 for char in text if char.isalnum() or char.isspace())
-    total_chars = len(text)
-    if total_chars > 0:
-        alphanumeric_ratio = alphanumeric_chars / total_chars
-        scores.append(alphanumeric_ratio)
-    
-    # Calculate final score
-    return sum(scores) / len(scores)
-
-
-def extract_keywords(text: str, language: str = 'english', max_keywords: int = 10) -> List[str]:
-    """
-    Extract keywords from text using TF-IDF approach.
-    
-    Args:
-        text: Text to analyze
-        language: Language for stopwords
-        max_keywords: Maximum number of keywords to return
-        
-    Returns:
-        List of keywords
-    """
-    if not text:
-        return []
-        
-    # Tokenize and clean
-    words = word_tokenize(text.lower())
-    
-    # Remove stopwords and short words
-    try:
-        stop_words = set(stopwords.words(language))
-    except:
-        stop_words = set()
-        
-    keywords = [
-        word for word in words 
-        if word.isalnum() and len(word) > 3 and word not in stop_words
-    ]
-    
-    # Count frequencies
-    word_freq = {}
-    for word in keywords:
-        word_freq[word] = word_freq.get(word, 0) + 1
-        
-    # Sort by frequency and return top keywords
-    sorted_keywords = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)
-    
-    return [word for word, _ in sorted_keywords[:max_keywords]]
-
-
-def truncate_text(text: str, max_length: int, ellipsis: str = "...") -> str:
-    """
-    Truncate text to maximum length, breaking at word boundaries.
-    
-    Args:
-        text: Text to truncate
-        max_length: Maximum length
-        ellipsis: String to append if truncated
-        
-    Returns:
-        Truncated text
-    """
-    if not text or len(text) <= max_length:
-        return text
-        
-    # Find last space before max_length
-    truncate_at = text.rfind(' ', 0, max_length - len(ellipsis))
-    
-    if truncate_at == -1:
-        # No space found, hard truncate
-        return text[:max_length - len(ellipsis)] + ellipsis
-        
-    return text[:truncate_at] + ellipsis
-
-
-def normalize_url(url: str) -> str:
-    """
-    Normalize URL for comparison and deduplication.
-    
-    Args:
-        url: URL to normalize
-        
-    Returns:
-        Normalized URL
-    """
-    if not url:
-        return ""
-        
-    # Remove trailing slashes
-    url = url.rstrip('/')
-    
-    # Remove common tracking parameters
-    tracking_params = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid']
-    
-    for param in tracking_params:
-        url = re.sub(rf'[?&]{param}=[^&]*', '', url)
-        
-    # Clean up multiple ? or &
-    url = re.sub(r'\?&', '?', url)
-    url = re.sub(r'&&+', '&', url)
-    url = url.rstrip('?&')
-    
-    return url
diff --git a/apps/backend/app/utils/validators.py b/apps/backend/app/utils/validators.py
deleted file mode 100644
index 01632fb..0000000
--- a/apps/backend/app/utils/validators.py
+++ /dev/null
@@ -1,367 +0,0 @@
-"""
-Validation utilities for the UnSearch API.
-"""
-import re
-from typing import List, Optional, Dict, Any
-from urllib.parse import urlparse
-import ipaddress
-from pydantic import validator
-
-
-def validate_query(query: str) -> str:
-    """
-    Validate and sanitize search query.
-    
-    Args:
-        query: Search query to validate
-        
-    Returns:
-        Validated query
-        
-    Raises:
-        ValueError: If query is invalid
-    """
-    if not query or not query.strip():
-        raise ValueError("Query cannot be empty")
-    
-    query = query.strip()
-    
-    if len(query) > 500:
-        raise ValueError("Query too long (max 500 characters)")
-    
-    # Check for suspicious patterns
-    suspicious_patterns = [
-        r']*>.*?',  # Script tags
-        r'javascript:',               # JavaScript URLs
-        r'vbscript:',                # VBScript URLs
-        r'onload\s*=',               # Event handlers
-        r'onerror\s*=',
-        r'onclick\s*=',
-        r'data:text/html',           # Data URLs
-        r'expression\s*\(',          # CSS expressions
-    ]
-    
-    for pattern in suspicious_patterns:
-        if re.search(pattern, query, re.IGNORECASE):
-            raise ValueError("Query contains potentially harmful content")
-    
-    return query
-
-
-def validate_url(url: str, allow_private: bool = False) -> str:
-    """
-    Validate URL for scraping.
-    
-    Args:
-        url: URL to validate
-        allow_private: Whether to allow private/local IPs
-        
-    Returns:
-        Validated URL
-        
-    Raises:
-        ValueError: If URL is invalid
-    """
-    if not url:
-        raise ValueError("URL cannot be empty")
-    
-    try:
-        parsed = urlparse(url)
-    except Exception:
-        raise ValueError("Invalid URL format")
-    
-    if not parsed.scheme:
-        raise ValueError("URL must include scheme (http/https)")
-    
-    if parsed.scheme not in ['http', 'https']:
-        raise ValueError("Only HTTP and HTTPS URLs are allowed")
-    
-    if not parsed.netloc:
-        raise ValueError("URL must include hostname")
-    
-    # Check for dangerous characters
-    if any(char in url for char in ['<', '>', '"', "'", '`']):
-        raise ValueError("URL contains invalid characters")
-    
-    # Validate hostname
-    hostname = parsed.hostname
-    if hostname:
-        # Check if it's an IP address
-        try:
-            ip = ipaddress.ip_address(hostname)
-            if not allow_private and (ip.is_private or ip.is_loopback or ip.is_reserved):
-                raise ValueError("Private/local IP addresses are not allowed")
-        except ValueError:
-            # Not an IP, validate hostname
-            if not re.match(r'^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?)*$', hostname):
-                raise ValueError("Invalid hostname format")
-    
-    return url
-
-
-def validate_engines(engines: List[str]) -> List[str]:
-    """
-    Validate search engines list.
-    
-    Args:
-        engines: List of engine names
-        
-    Returns:
-        Validated engines list
-        
-    Raises:
-        ValueError: If engines list is invalid
-    """
-    if not engines:
-        raise ValueError("At least one search engine must be specified")
-    
-    allowed_engines = {
-        'google', 'bing', 'duckduckgo', 'startpage', 'qwant',
-        'yahoo', 'searx', 'brave', 'ecosia', 'yandex'
-    }
-    
-    # Validate each engine
-    validated = []
-    for engine in engines:
-        if not isinstance(engine, str):
-            raise ValueError("Engine names must be strings")
-        
-        engine = engine.lower().strip()
-        if not engine:
-            continue
-            
-        if engine not in allowed_engines:
-            raise ValueError(f"Unsupported search engine: {engine}")
-        
-        if engine not in validated:
-            validated.append(engine)
-    
-    if not validated:
-        raise ValueError("No valid search engines provided")
-    
-    return validated
-
-
-def validate_language_code(language: str) -> str:
-    """
-    Validate ISO 639-1 language code.
-    
-    Args:
-        language: Language code to validate
-        
-    Returns:
-        Validated language code
-        
-    Raises:
-        ValueError: If language code is invalid
-    """
-    if not isinstance(language, str):
-        raise ValueError("Language code must be a string")
-    
-    language = language.lower().strip()
-    
-    if not re.match(r'^[a-z]{2}$', language):
-        raise ValueError("Language code must be a 2-letter ISO 639-1 code")
-    
-    # Common language codes (not exhaustive, but covers main ones)
-    valid_codes = {
-        'en', 'es', 'fr', 'de', 'it', 'pt', 'ru', 'zh', 'ja', 'ko',
-        'ar', 'hi', 'tr', 'pl', 'nl', 'sv', 'da', 'no', 'fi', 'he',
-        'th', 'vi', 'id', 'ms', 'tl', 'sw', 'am', 'bn', 'ta', 'te',
-        'ml', 'kn', 'gu', 'or', 'pa', 'as', 'ne', 'si', 'my', 'km'
-    }
-    
-    if language not in valid_codes:
-        # Allow it but log a warning
-        pass
-    
-    return language
-
-
-def validate_css_selector(selector: str) -> str:
-    """
-    Validate CSS selector for safety.
-    
-    Args:
-        selector: CSS selector to validate
-        
-    Returns:
-        Validated selector
-        
-    Raises:
-        ValueError: If selector is invalid or dangerous
-    """
-    if not selector or not selector.strip():
-        raise ValueError("CSS selector cannot be empty")
-    
-    selector = selector.strip()
-    
-    if len(selector) > 200:
-        raise ValueError("CSS selector too long (max 200 characters)")
-    
-    # Check for dangerous patterns
-    dangerous_patterns = [
-        r'javascript:',
-        r'expression\s*\(',
-        r'behavior\s*:',
-        r'@import',
-        r'url\s*\(',
-        r'<.*?>',
-    ]
-    
-    for pattern in dangerous_patterns:
-        if re.search(pattern, selector, re.IGNORECASE):
-            raise ValueError("CSS selector contains potentially harmful content")
-    
-    # Basic CSS selector validation
-    if not re.match(r'^[a-zA-Z0-9\s\.\#\[\]\(\)\:\-\*\>\+\~\,\"\'=\|\_]+$', selector):
-        raise ValueError("CSS selector contains invalid characters")
-    
-    return selector
-
-
-def validate_custom_selectors(selectors: Dict[str, str]) -> Dict[str, str]:
-    """
-    Validate custom CSS selectors dictionary.
-    
-    Args:
-        selectors: Dictionary of custom selectors
-        
-    Returns:
-        Validated selectors
-        
-    Raises:
-        ValueError: If selectors are invalid
-    """
-    if not isinstance(selectors, dict):
-        raise ValueError("Custom selectors must be a dictionary")
-    
-    if len(selectors) > 20:
-        raise ValueError("Too many custom selectors (max 20)")
-    
-    validated = {}
-    
-    for field, selector in selectors.items():
-        # Validate field name
-        if not isinstance(field, str) or not field.strip():
-            raise ValueError("Selector field names must be non-empty strings")
-        
-        field = field.strip()
-        if len(field) > 50:
-            raise ValueError(f"Selector field name too long: {field}")
-        
-        if not re.match(r'^[a-zA-Z0-9_]+$', field):
-            raise ValueError(f"Invalid field name: {field}")
-        
-        # Validate selector
-        validated[field] = validate_css_selector(selector)
-    
-    return validated
-
-
-def validate_webhook_url(url: str) -> str:
-    """
-    Validate webhook URL.
-    
-    Args:
-        url: Webhook URL to validate
-        
-    Returns:
-        Validated URL
-        
-    Raises:
-        ValueError: If URL is invalid for webhooks
-    """
-    validated_url = validate_url(url, allow_private=False)
-    
-    # Additional webhook-specific validations
-    parsed = urlparse(validated_url)
-    
-    # Must be HTTPS in production
-    if parsed.scheme != 'https':
-        # Allow HTTP for development/testing
-        pass
-    
-    # Check path doesn't contain suspicious elements
-    if parsed.path:
-        if any(suspicious in parsed.path.lower() for suspicious in ['admin', 'config', 'internal']):
-            raise ValueError("Webhook URL path appears to target internal endpoints")
-    
-    return validated_url
-
-
-def validate_timeout(timeout: int, min_timeout: int = 5, max_timeout: int = 120) -> int:
-    """
-    Validate timeout value.
-    
-    Args:
-        timeout: Timeout in seconds
-        min_timeout: Minimum allowed timeout
-        max_timeout: Maximum allowed timeout
-        
-    Returns:
-        Validated timeout
-        
-    Raises:
-        ValueError: If timeout is invalid
-    """
-    if not isinstance(timeout, int):
-        raise ValueError("Timeout must be an integer")
-    
-    if timeout < min_timeout:
-        raise ValueError(f"Timeout too low (minimum {min_timeout} seconds)")
-    
-    if timeout > max_timeout:
-        raise ValueError(f"Timeout too high (maximum {max_timeout} seconds)")
-    
-    return timeout
-
-
-def validate_cache_ttl(ttl: int) -> int:
-    """
-    Validate cache TTL value.
-    
-    Args:
-        ttl: TTL in seconds
-        
-    Returns:
-        Validated TTL
-        
-    Raises:
-        ValueError: If TTL is invalid
-    """
-    if not isinstance(ttl, int):
-        raise ValueError("Cache TTL must be an integer")
-    
-    if ttl < 0:
-        raise ValueError("Cache TTL cannot be negative")
-    
-    if ttl > 86400:  # 24 hours
-        raise ValueError("Cache TTL too high (maximum 24 hours)")
-    
-    return ttl
-
-
-def validate_max_results(max_results: int) -> int:
-    """
-    Validate max results parameter.
-    
-    Args:
-        max_results: Maximum number of results
-        
-    Returns:
-        Validated max results
-        
-    Raises:
-        ValueError: If max results is invalid
-    """
-    if not isinstance(max_results, int):
-        raise ValueError("Max results must be an integer")
-    
-    if max_results < 1:
-        raise ValueError("Max results must be at least 1")
-    
-    if max_results > 100:
-        raise ValueError("Max results too high (maximum 100)")
-    
-    return max_results
diff --git a/apps/backend/app/workers/__init__.py b/apps/backend/app/workers/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/apps/backend/app/workers/tasks.py b/apps/backend/app/workers/tasks.py
deleted file mode 100644
index 9e9de94..0000000
--- a/apps/backend/app/workers/tasks.py
+++ /dev/null
@@ -1,236 +0,0 @@
-"""
-Celery tasks for async processing.
-"""
-from celery import Celery, Task
-from typing import Dict, Any
-import httpx
-import asyncio
-from datetime import datetime
-
-from app.config import get_settings
-from app.services.searxng import SearXNGService
-from app.services.scraping import ContentScrapingService
-from app.services.database import DatabaseService
-from app.models.requests import UnQuestRequest, ScrapingConfig
-import structlog
-
-logger = structlog.get_logger(__name__)
-settings = get_settings()
-
-# Initialize Celery
-celery = Celery(
-    'UnQuest',
-    broker=settings.celery_broker_url,
-    backend=settings.celery_result_backend
-)
-
-# Configure Celery
-celery.conf.update(
-    task_serializer='json',
-    accept_content=['json'],
-    result_serializer='json',
-    timezone='UTC',
-    enable_utc=True,
-    task_soft_time_limit=settings.celery_task_soft_time_limit,
-    task_time_limit=settings.celery_task_time_limit,
-    worker_concurrency=settings.celery_worker_concurrency,
-    worker_prefetch_multiplier=1,
-    task_acks_late=True,
-    task_reject_on_worker_lost=True
-)
-
-
-class AsyncTask(Task):
-    """Base task that properly handles async operations."""
-    
-    def run(self, *args, **kwargs):
-        """Run the task in an async context."""
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        try:
-            return loop.run_until_complete(self.async_run(*args, **kwargs))
-        finally:
-            loop.close()
-            
-    async def async_run(self, *args, **kwargs):
-        """Override this method in subclasses."""
-        raise NotImplementedError
-
-
-@celery.task(base=AsyncTask, bind=True, max_retries=3)
-async def process_async_search_scrape(self, job_id: str, request_data: Dict[str, Any]):
-    """
-    Process search and scrape request asynchronously.
-    
-    Args:
-        job_id: Scraping job ID
-        request_data: UnQuestRequest data as dict
-    """
-    logger.info("async_search_scrape_started", job_id=job_id)
-    
-    try:
-        # Initialize services
-        searxng = SearXNGService()
-        scraper = ContentScrapingService()
-        db = DatabaseService()
-        
-        await searxng.initialize()
-        await scraper.initialize()
-        await db.initialize()
-        
-        # Update job status
-        await db.update_scraping_job(
-            job_id,
-            status="processing",
-            task_id=self.request.id
-        )
-        
-        # Create request object
-        request = UnQuestRequest(**request_data)
-        
-        # Perform search
-        search_results = await searxng.search(
-            query=request.query,
-            engines=request.engines,
-            language=request.language,
-            safe_search=1 if request.safe_search == "moderate" else 0
-        )
-        
-        # Limit results
-        search_results = search_results[:request.max_results]
-        
-        # Scrape content if requested
-        scraped_results = []
-        if request.scrape_content and search_results:
-            urls = [str(result.url) for result in search_results]
-            
-            scraping_config = ScrapingConfig(
-                urls=urls,
-                selectors=request.scrape_selectors,
-                extract_images=request.include_images,
-                extract_links=request.include_links,
-                javascript_rendering=request.js_mode,
-                js_mode=request.js_mode,
-                response_format=request.output_format,
-                screenshot=request.screenshot,
-                pdf=request.pdf,
-            )
-            
-            scraped_contents = await scraper.scrape_urls(urls[:10], scraping_config)
-            
-            # Combine results
-            scraped_map = {str(sc.url): sc for sc in scraped_contents}
-            
-            for result in search_results:
-                result_dict = result.dict()
-                if str(result.url) in scraped_map:
-                    result_dict['scraped_content'] = scraped_map[str(result.url)].dict()
-                scraped_results.append(result_dict)
-        else:
-            scraped_results = [r.dict() for r in search_results]
-            
-        # Update job with results
-        await db.update_scraping_job(
-            job_id,
-            status="completed",
-            results=scraped_results
-        )
-        
-        # Send webhook if configured
-        if request_data.get('webhook_url'):
-            await send_webhook(
-                job_id,
-                request_data['webhook_url'],
-                scraped_results,
-                db
-            )
-            
-        logger.info("async_search_scrape_completed", job_id=job_id, results_count=len(scraped_results))
-        
-        # Cleanup
-        await searxng.close()
-        await scraper.close()
-        await db.close()
-        
-        return {"job_id": job_id, "status": "completed", "results_count": len(scraped_results)}
-        
-    except Exception as e:
-        logger.error("async_search_scrape_error", job_id=job_id, error=str(e))
-        
-        # Update job status
-        try:
-            db = DatabaseService()
-            await db.initialize()
-            await db.update_scraping_job(
-                job_id,
-                status="failed",
-                error_message=str(e)
-            )
-            await db.close()
-        except:
-            pass
-            
-        # Retry with exponential backoff
-        raise self.retry(exc=e, countdown=2 ** self.request.retries)
-
-
-@celery.task(base=AsyncTask)
-async def send_webhook(job_id: str, webhook_url: str, results: list, db: DatabaseService):
-    """
-    Send webhook notification with results.
-    
-    Args:
-        job_id: Job ID
-        webhook_url: URL to send results to
-        results: Search/scrape results
-        db: Database service instance
-    """
-    payload = {
-        "job_id": job_id,
-        "status": "completed",
-        "results": results,
-        "completed_at": datetime.utcnow().isoformat()
-    }
-    
-    try:
-        async with httpx.AsyncClient(timeout=30) as client:
-            response = await client.post(
-                webhook_url,
-                json=payload,
-                headers={"Content-Type": "application/json"}
-            )
-            response.raise_for_status()
-            
-        await db.update_scraping_job(
-            job_id,
-            status="completed",
-            webhook_success=True
-        )
-        
-        logger.info("webhook_sent", job_id=job_id, webhook_url=webhook_url)
-        
-    except Exception as e:
-        logger.error("webhook_error", job_id=job_id, webhook_url=webhook_url, error=str(e))
-        
-        await db.update_scraping_job(
-            job_id,
-            status="completed",
-            webhook_success=False
-        )
-
-
-@celery.task
-def cleanup_old_jobs():
-    """Periodic task to cleanup old scraping jobs."""
-    logger.info("cleanup_old_jobs_started")
-    # Implementation would go here
-    pass
-
-
-# Configure periodic tasks
-celery.conf.beat_schedule = {
-    'cleanup-old-jobs': {
-        'task': 'app.workers.tasks.cleanup_old_jobs',
-        'schedule': 3600.0,  # Every hour
-    },
-}
diff --git a/apps/backend/docker-compose.prod.yml b/apps/backend/docker-compose.prod.yml
deleted file mode 100644
index 5473750..0000000
--- a/apps/backend/docker-compose.prod.yml
+++ /dev/null
@@ -1,88 +0,0 @@
-version: '3.8'
-
-# Production overrides for docker-compose.yml
-services:
-  api:
-    environment:
-      - ENVIRONMENT=production
-      - DEBUG=False
-      - LOG_LEVEL=WARNING
-      - WORKERS=8
-    deploy:
-      resources:
-        limits:
-          cpus: '2'
-          memory: 2G
-        reservations:
-          cpus: '1'
-          memory: 1G
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "100m"
-        max-file: "5"
-
-  searxng:
-    environment:
-      - SEARXNG_SECRET_KEY=${SEARXNG_SECRET_KEY}
-    deploy:
-      resources:
-        limits:
-          cpus: '1'
-          memory: 1G
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "50m"
-        max-file: "3"
-
-  redis:
-    command: redis-server --appendonly yes --maxmemory 1gb --maxmemory-policy allkeys-lru --requirepass ${REDIS_PASSWORD:-}
-    deploy:
-      resources:
-        limits:
-          cpus: '0.5'
-          memory: 1.5G
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "50m"
-        max-file: "3"
-
-  postgres:
-    environment:
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    deploy:
-      resources:
-        limits:
-          cpus: '1'
-          memory: 2G
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "50m"
-        max-file: "3"
-
-  celery-worker:
-    deploy:
-      replicas: 2
-      resources:
-        limits:
-          cpus: '1'
-          memory: 1G
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "50m"
-        max-file: "3"
-
-  nginx:
-    volumes:
-      - ./nginx/nginx.prod.conf:/etc/nginx/nginx.conf:ro
-      - ./nginx/ssl:/etc/nginx/ssl:ro
-      - /etc/letsencrypt:/etc/letsencrypt:ro
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "50m"
-        max-file: "3"
diff --git a/apps/backend/docker-compose.yml b/apps/backend/docker-compose.yml
deleted file mode 100644
index 1fa220b..0000000
--- a/apps/backend/docker-compose.yml
+++ /dev/null
@@ -1,174 +0,0 @@
-version: '3.8'
-
-services:
-  api:
-    build: .
-    container_name: unsearch-api
-    ports:
-      - "8000:8000"
-    environment:
-      - ENVIRONMENT=production
-      - SEARXNG_URL=http://searxng:8080
-      - REDIS_URL=${REDIS_URL}
-      - DATABASE_URL=${DATABASE_URL}
-      - API_KEYS=${API_KEYS:-}
-      - LOG_LEVEL=INFO
-      - WORKERS=4
-    depends_on:
-      searxng:
-        condition: service_healthy
-      # redis:  # Using Upstash Redis instead
-      #   condition: service_healthy
-      postgres:
-        condition: service_healthy
-    networks:
-      - unsearch-net
-    volumes:
-      - ./logs:/app/logs
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 40s
-    restart: unless-stopped
-
-  searxng:
-    image: searxng/searxng:latest
-    container_name: unsearch-searxng
-    ports:
-      - "8080:8080"
-    environment:
-      - SEARXNG_BASE_URL=http://searxng:8080/
-      - SEARXNG_SECRET_KEY=${SEARXNG_SECRET_KEY:-ultrasecretkey}
-    volumes:
-      - ./searxng:/etc/searxng:rw
-    networks:
-      - unsearch-net
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8080/healthz"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-    restart: unless-stopped
-    cap_drop:
-      - ALL
-    cap_add:
-      - CHOWN
-      - SETGID
-      - SETUID
-      - DAC_OVERRIDE
-
-  # Redis service disabled - using Upstash Redis cloud service
-  # redis:
-  #   image: redis:7-alpine
-  #   container_name: unsearch-redis
-  #   ports:
-  #     - "6379:6379"
-  #   command: redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru
-  #   volumes:
-  #     - redis_data:/data
-  #   networks:
-  #     - UnSearch-net
-  #   healthcheck:
-  #     test: ["CMD", "redis-cli", "ping"]
-  #     interval: 10s
-  #     timeout: 5s
-  #     retries: 5
-  #   restart: unless-stopped
-
-  postgres:
-    image: postgres:15-alpine
-    container_name: unsearch-postgres
-    ports:
-      - "5432:5432"
-    environment:
-      POSTGRES_DB: unsearch
-      POSTGRES_USER: unsearch
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-changeme}
-      PGDATA: /var/lib/postgresql/data/pgdata
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    networks:
-      - unsearch-net
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U unsearch"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    restart: unless-stopped
-
-  celery-worker:
-    build: .
-    container_name: unsearch-celery-worker
-    command: celery -A app.workers.tasks worker --loglevel=info --concurrency=4
-    environment:
-      - SEARXNG_URL=http://searxng:8080
-      - REDIS_URL=${REDIS_URL}
-      - DATABASE_URL=${DATABASE_URL}
-      - CELERY_BROKER_URL=${CELERY_BROKER_URL}
-      - CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
-    depends_on:
-      # - redis  # Using Upstash Redis instead
-      - postgres
-      - searxng
-    networks:
-      - unsearch-net
-    volumes:
-      - ./logs:/app/logs
-    restart: unless-stopped
-
-  celery-beat:
-    build: .
-    container_name: unsearch-celery-beat
-    command: celery -A app.workers.tasks beat --loglevel=info
-    environment:
-      - REDIS_URL=${REDIS_URL}
-      - DATABASE_URL=${DATABASE_URL}
-      - CELERY_BROKER_URL=${CELERY_BROKER_URL}
-      - CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
-    depends_on:
-      # - redis  # Using Upstash Redis instead
-      - postgres
-    networks:
-      - unsearch-net
-    restart: unless-stopped
-
-  flower:
-    build: .
-    container_name: unsearch-flower
-    command: celery -A app.workers.tasks flower --port=5555
-    ports:
-      - "5555:5555"
-    environment:
-      - CELERY_BROKER_URL=${CELERY_BROKER_URL}
-      - CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND}
-    depends_on:
-      # - redis  # Using Upstash Redis instead
-      - celery-worker
-    networks:
-      - unsearch-net
-    restart: unless-stopped
-
-  nginx:
-    image: nginx:alpine
-    container_name: unsearch-nginx
-    ports:
-      - "80:80"
-      - "443:443"
-    volumes:
-      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
-      - ./nginx/ssl:/etc/nginx/ssl:ro
-    depends_on:
-      - api
-    networks:
-      - unsearch-net
-    restart: unless-stopped
-
-networks:
-  unsearch-net:
-    driver: bridge
-
-volumes:
-  # redis_data:  # Not needed when using Upstash Redis
-  postgres_data:
diff --git a/apps/backend/env.example b/apps/backend/env.example
deleted file mode 100644
index 4ab7320..0000000
--- a/apps/backend/env.example
+++ /dev/null
@@ -1,57 +0,0 @@
-# Backend Environment Variables Example
-# Copy this file to .env and update with your values
-
-# Database Configuration
-DATABASE_URL=postgresql://postgres:postgres@localhost:5432/unsearch
-# For production: postgresql://user:password@host:port/database?sslmode=require
-
-# Redis Configuration
-REDIS_URL=redis://localhost:6379
-# For production with auth: redis://:password@redis-host:6379/0
-
-# Security Keys (REQUIRED - Generate new keys!)
-SECRET_KEY=change-this-to-a-random-32-character-string
-# Generate with: openssl rand -hex 32
-JWT_SECRET_KEY=change-this-to-another-random-32-char-string
-# Generate with: openssl rand -hex 32
-
-# Environment
-ENVIRONMENT=development
-DEBUG=true
-
-# API Configuration
-APP_NAME=UnSearch API
-VERSION=1.0.0
-API_PREFIX=/api/v1
-
-# CORS Configuration
-ALLOWED_ORIGINS=["http://localhost:3000"]
-CORS_CREDENTIALS=true
-CORS_METHODS=["GET","POST","PUT","DELETE","OPTIONS"]
-CORS_HEADERS=["*"]
-
-# Stripe Configuration (Optional for development)
-STRIPE_SECRET_KEY=sk_test_...
-STRIPE_PUBLISHABLE_KEY=pk_test_...
-STRIPE_WEBHOOK_SECRET=whsec_...
-STRIPE_PRO_PRICE_ID=price_...
-
-# SearXNG Configuration
-SEARXNG_URL=http://localhost:8080
-SEARXNG_SECRET=change-me-with-openssl-rand-hex-32
-
-# Rate Limiting
-RATE_LIMIT_ENABLED=true
-RATE_LIMIT_DEFAULT=100/minute
-RATE_LIMIT_STORAGE_URL=redis://localhost:6379/1
-
-# Celery Configuration (Optional)
-CELERY_BROKER_URL=redis://localhost:6379/2
-CELERY_RESULT_BACKEND=redis://localhost:6379/3
-
-# Email Configuration (Optional)
-SMTP_HOST=smtp.gmail.com
-SMTP_PORT=587
-SMTP_USER=your-email@example.com
-SMTP_PASSWORD=your-app-password
-SMTP_FROM=noreply@unsearch.dev
diff --git a/apps/backend/monitoring/docker-compose.monitoring.yml b/apps/backend/monitoring/docker-compose.monitoring.yml
deleted file mode 100644
index c58a306..0000000
--- a/apps/backend/monitoring/docker-compose.monitoring.yml
+++ /dev/null
@@ -1,134 +0,0 @@
-version: '3.8'
-
-services:
-  prometheus:
-    image: prom/prometheus:latest
-    container_name: unsearch-prometheus
-    ports:
-      - "9090:9090"
-    volumes:
-      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - prometheus_data:/prometheus
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
-      - '--web.enable-lifecycle'
-      - '--storage.tsdb.retention.time=30d'
-    networks:
-      - monitoring-net
-    restart: unless-stopped
-
-  grafana:
-    image: grafana/grafana:latest
-    container_name: unsearch-grafana
-    ports:
-      - "3000:3000"
-    environment:
-      - GF_SECURITY_ADMIN_USER=${GF_ADMIN_USER:-admin}
-      - GF_SECURITY_ADMIN_PASSWORD=${GF_ADMIN_PASSWORD:-changeme}
-      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,redis-datasource
-      - GF_SERVER_ROOT_URL=http://localhost:3000
-      - GF_AUTH_ANONYMOUS_ENABLED=false
-    volumes:
-      - grafana_data:/var/lib/grafana
-      - ./grafana/provisioning:/etc/grafana/provisioning:ro
-      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
-    networks:
-      - monitoring-net
-    restart: unless-stopped
-    depends_on:
-      - prometheus
-
-  node-exporter:
-    image: prom/node-exporter:latest
-    container_name: unsearch-node-exporter
-    ports:
-      - "9100:9100"
-    volumes:
-      - /proc:/host/proc:ro
-      - /sys:/host/sys:ro
-      - /:/rootfs:ro
-    command:
-      - '--path.procfs=/host/proc'
-      - '--path.sysfs=/host/sys'
-      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
-    networks:
-      - monitoring-net
-    restart: unless-stopped
-
-  postgres-exporter:
-    image: prometheuscommunity/postgres-exporter:latest
-    container_name: unsearch-postgres-exporter
-    ports:
-      - "9187:9187"
-    environment:
-      DATA_SOURCE_NAME: "postgresql://${POSTGRES_USER:-unsearch}:${POSTGRES_PASSWORD:-changeme}@postgres:5432/${POSTGRES_DB:-unsearch}?sslmode=disable"
-    networks:
-      - monitoring-net
-      - unsearch-net
-    restart: unless-stopped
-
-  redis-exporter:
-    image: oliver006/redis_exporter:latest
-    container_name: unsearch-redis-exporter
-    ports:
-      - "9121:9121"
-    environment:
-      REDIS_ADDR: "redis://redis:6379"
-    networks:
-      - monitoring-net
-      - unsearch-net
-    restart: unless-stopped
-
-  alertmanager:
-    image: prom/alertmanager:latest
-    container_name: unsearch-alertmanager
-    ports:
-      - "9093:9093"
-    volumes:
-      - ./prometheus/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
-      - alertmanager_data:/alertmanager
-    command:
-      - '--config.file=/etc/alertmanager/alertmanager.yml'
-      - '--storage.path=/alertmanager'
-    networks:
-      - monitoring-net
-    restart: unless-stopped
-
-  loki:
-    image: grafana/loki:latest
-    container_name: unsearch-loki
-    ports:
-      - "3100:3100"
-    volumes:
-      - ./loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
-      - loki_data:/loki
-    networks:
-      - monitoring-net
-    restart: unless-stopped
-
-  promtail:
-    image: grafana/promtail:latest
-    container_name: unsearch-promtail
-    volumes:
-      - ./promtail/promtail-config.yaml:/etc/promtail/config.yml:ro
-      - /var/log:/var/log:ro
-      - ../logs:/app/logs:ro
-    command: -config.file=/etc/promtail/config.yml
-    networks:
-      - monitoring-net
-    restart: unless-stopped
-
-networks:
-  monitoring-net:
-    driver: bridge
-  unsearch-net:
-    external: true
-
-volumes:
-  prometheus_data:
-  grafana_data:
-  alertmanager_data:
-  loki_data:
diff --git a/apps/backend/monitoring/grafana/dashboards/searchscrape-overview.json b/apps/backend/monitoring/grafana/dashboards/searchscrape-overview.json
deleted file mode 100644
index 2e913d8..0000000
--- a/apps/backend/monitoring/grafana/dashboards/searchscrape-overview.json
+++ /dev/null
@@ -1,627 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": "-- Grafana --",
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "gnetId": null,
-  "graphTooltip": 1,
-  "id": 1,
-  "links": [],
-  "panels": [
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "red",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 0.95
-              },
-              {
-                "color": "green",
-                "value": 0.99
-              }
-            ]
-          },
-          "unit": "percentunit"
-        }
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "text": {}
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "up{job=\"UnSearch-api\"}",
-          "refId": "A"
-        }
-      ],
-      "title": "API Uptime",
-      "type": "gauge"
-    },
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "tooltip": false,
-              "viz": false,
-              "legend": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": true,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "reqps"
-        }
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 6,
-        "y": 0
-      },
-      "id": 2,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom"
-        },
-        "tooltip": {
-          "mode": "single"
-        }
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "rate(http_requests_total{job=\"UnSearch-api\"}[5m])",
-          "legendFormat": "{{method}} {{endpoint}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Request Rate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 100
-              },
-              {
-                "color": "red",
-                "value": 500
-              }
-            ]
-          },
-          "unit": "ms"
-        }
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 18,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": ["mean"],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "text": {}
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket{job=\"UnSearch-api\"}[5m])) * 1000",
-          "refId": "A"
-        }
-      ],
-      "title": "P95 Response Time",
-      "type": "gauge"
-    },
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "decimals": 0,
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "short"
-        }
-      },
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 4
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": ["sum"],
-          "fields": "",
-          "values": false
-        },
-        "text": {},
-        "textMode": "auto"
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "increase(http_requests_total{job=\"UnSearch-api\"}[1h])",
-          "refId": "A"
-        }
-      ],
-      "title": "Total Requests (1h)",
-      "type": "stat"
-    },
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "tooltip": false,
-              "viz": false,
-              "legend": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": true,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "ms"
-        }
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 8
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": ["mean", "max", "min"],
-          "displayMode": "table",
-          "placement": "bottom"
-        },
-        "tooltip": {
-          "mode": "single"
-        }
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "histogram_quantile(0.5, rate(http_request_duration_seconds_bucket{job=\"UnSearch-api\"}[5m])) * 1000",
-          "legendFormat": "P50",
-          "refId": "A"
-        },
-        {
-          "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket{job=\"UnSearch-api\"}[5m])) * 1000",
-          "legendFormat": "P95",
-          "refId": "B"
-        },
-        {
-          "expr": "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{job=\"UnSearch-api\"}[5m])) * 1000",
-          "legendFormat": "P99",
-          "refId": "C"
-        }
-      ],
-      "title": "Response Time Percentiles",
-      "type": "timeseries"
-    },
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "tooltip": false,
-              "viz": false,
-              "legend": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": true,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "percentunit"
-        }
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 8
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom"
-        },
-        "tooltip": {
-          "mode": "single"
-        }
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "rate(http_requests_total{job=\"UnSearch-api\",status=~\"5..\"}[5m]) / rate(http_requests_total{job=\"UnSearch-api\"}[5m])",
-          "legendFormat": "Error Rate",
-          "refId": "A"
-        }
-      ],
-      "title": "Error Rate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "tooltip": false,
-              "viz": false,
-              "legend": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": true,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "short"
-        }
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 16
-      },
-      "id": 7,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom"
-        },
-        "tooltip": {
-          "mode": "single"
-        }
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "rate(search_requests_total{job=\"UnSearch-api\"}[5m])",
-          "legendFormat": "Search Requests",
-          "refId": "A"
-        }
-      ],
-      "title": "Search Request Rate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "red",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 0.5
-              },
-              {
-                "color": "green",
-                "value": 0.8
-              }
-            ]
-          },
-          "unit": "percentunit"
-        }
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 6,
-        "x": 12,
-        "y": 16
-      },
-      "id": 8,
-      "options": {
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "text": {}
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "cache_hit_ratio",
-          "refId": "A"
-        }
-      ],
-      "title": "Cache Hit Ratio",
-      "type": "gauge"
-    },
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "red",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 0.7
-              },
-              {
-                "color": "green",
-                "value": 0.9
-              }
-            ]
-          },
-          "unit": "percentunit"
-        }
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 6,
-        "x": 18,
-        "y": 16
-      },
-      "id": 9,
-      "options": {
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "text": {}
-      },
-      "pluginVersion": "8.0.0",
-      "targets": [
-        {
-          "expr": "scraping_success_rate",
-          "refId": "A"
-        }
-      ],
-      "title": "Scraping Success Rate",
-      "type": "gauge"
-    }
-  ],
-  "refresh": "10s",
-  "schemaVersion": 27,
-  "style": "dark",
-  "tags": ["UnSearch", "api", "monitoring"],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "UnSearch API Overview",
-  "uid": "UnSearch-overview",
-  "version": 1
-}
diff --git a/apps/backend/monitoring/grafana/provisioning/dashboards/dashboards.yml b/apps/backend/monitoring/grafana/provisioning/dashboards/dashboards.yml
deleted file mode 100644
index 1371e4b..0000000
--- a/apps/backend/monitoring/grafana/provisioning/dashboards/dashboards.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-apiVersion: 1
-
-providers:
-  - name: 'UnSearch Dashboards'
-    orgId: 1
-    folder: 'UnSearch'
-    folderUid: 'UnSearch'
-    type: file
-    disableDeletion: false
-    updateIntervalSeconds: 10
-    allowUiUpdates: true
-    options:
-      path: /var/lib/grafana/dashboards
diff --git a/apps/backend/monitoring/grafana/provisioning/datasources/prometheus.yml b/apps/backend/monitoring/grafana/provisioning/datasources/prometheus.yml
deleted file mode 100644
index 424cb6c..0000000
--- a/apps/backend/monitoring/grafana/provisioning/datasources/prometheus.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    access: proxy
-    url: http://prometheus:9090
-    isDefault: true
-    editable: true
-    jsonData:
-      timeInterval: "15s"
-      queryTimeout: "60s"
-      httpMethod: "POST"
-
-  - name: Loki
-    type: loki
-    access: proxy
-    url: http://loki:3100
-    editable: true
-    jsonData:
-      maxLines: 1000
-
-  - name: Redis
-    type: redis-datasource
-    access: proxy
-    url: redis://redis:6379
-    editable: true
-    jsonData:
-      poolSize: 10
-      timeout: "10s"
-      pingInterval: "30s"
-      pipelineWindow: "0"
diff --git a/apps/backend/monitoring/prometheus/prometheus.yml b/apps/backend/monitoring/prometheus/prometheus.yml
deleted file mode 100644
index 989527d..0000000
--- a/apps/backend/monitoring/prometheus/prometheus.yml
+++ /dev/null
@@ -1,97 +0,0 @@
-global:
-  scrape_interval: 15s
-  evaluation_interval: 15s
-  external_labels:
-    monitor: 'UnSearch-monitor'
-    environment: 'production'
-
-# Alerting configuration
-alerting:
-  alertmanagers:
-    - static_configs:
-        - targets:
-            - alertmanager:9093
-
-# Load rules once and periodically evaluate them
-rule_files:
-  - "alerts.yml"
-
-# Scrape configurations
-scrape_configs:
-  # UnSearch API metrics
-  - job_name: 'UnSearch-api'
-    static_configs:
-      - targets: ['api:8000']
-        labels:
-          service: 'api'
-          environment: 'production'
-    metrics_path: '/metrics'
-    scrape_interval: 10s
-
-  # Prometheus self-monitoring
-  - job_name: 'prometheus'
-    static_configs:
-      - targets: ['localhost:9090']
-        labels:
-          service: 'prometheus'
-
-  # Node exporter for system metrics
-  - job_name: 'node-exporter'
-    static_configs:
-      - targets: ['node-exporter:9100']
-        labels:
-          service: 'node'
-
-  # PostgreSQL exporter
-  - job_name: 'postgres'
-    static_configs:
-      - targets: ['postgres-exporter:9187']
-        labels:
-          service: 'postgres'
-          database: 'UnSearch'
-
-  # Redis exporter
-  - job_name: 'redis'
-    static_configs:
-      - targets: ['redis-exporter:9121']
-        labels:
-          service: 'redis'
-
-  # SearXNG monitoring
-  - job_name: 'searxng'
-    static_configs:
-      - targets: ['searxng:8080']
-        labels:
-          service: 'searxng'
-    metrics_path: '/stats/prometheus'
-    scrape_interval: 30s
-
-  # Celery worker metrics (if exposed)
-  - job_name: 'celery-worker'
-    static_configs:
-      - targets: ['celery-worker:9540']
-        labels:
-          service: 'celery-worker'
-
-  # Flower metrics
-  - job_name: 'flower'
-    static_configs:
-      - targets: ['flower:5555']
-        labels:
-          service: 'flower'
-    metrics_path: '/metrics'
-
-  # Grafana metrics
-  - job_name: 'grafana'
-    static_configs:
-      - targets: ['grafana:3000']
-        labels:
-          service: 'grafana'
-    metrics_path: '/metrics'
-
-  # Nginx metrics (if nginx-prometheus-exporter is used)
-  - job_name: 'nginx'
-    static_configs:
-      - targets: ['nginx:9113']
-        labels:
-          service: 'nginx'
diff --git a/apps/backend/nginx/nginx.conf b/apps/backend/nginx/nginx.conf
deleted file mode 100644
index d7a9b87..0000000
--- a/apps/backend/nginx/nginx.conf
+++ /dev/null
@@ -1,145 +0,0 @@
-events {
-    worker_connections 1024;
-}
-
-http {
-    upstream api {
-        server api:8000;
-    }
-
-    upstream searxng {
-        server searxng:8080;
-    }
-
-    upstream flower {
-        server flower:5555;
-    }
-
-    # Rate limiting zones
-    limit_req_zone $binary_remote_addr zone=api_limit:10m rate=100r/s;
-    limit_req_zone $binary_remote_addr zone=search_limit:10m rate=10r/s;
-
-    # Gzip compression
-    gzip on;
-    gzip_types text/plain application/json application/xml text/css text/javascript application/javascript;
-    gzip_min_length 1000;
-
-    # Security headers
-    add_header X-Frame-Options "SAMEORIGIN" always;
-    add_header X-Content-Type-Options "nosniff" always;
-    add_header X-XSS-Protection "1; mode=block" always;
-    add_header Referrer-Policy "strict-origin-when-cross-origin" always;
-
-    # API server
-    server {
-        listen 80;
-        server_name api.UnSearch.io localhost;
-
-        # Redirect to HTTPS in production
-        # return 301 https://$server_name$request_uri;
-
-        # API endpoints
-        location /api/ {
-            limit_req zone=api_limit burst=20 nodelay;
-            
-            proxy_pass http://api;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-            
-            # Timeouts
-            proxy_connect_timeout 30s;
-            proxy_send_timeout 120s;
-            proxy_read_timeout 120s;
-            
-            # Buffering
-            proxy_buffering on;
-            proxy_buffer_size 4k;
-            proxy_buffers 8 4k;
-            proxy_busy_buffers_size 8k;
-        }
-
-        # Health check
-        location /health {
-            proxy_pass http://api/health;
-            access_log off;
-        }
-
-        # Metrics (restricted)
-        location /metrics {
-            proxy_pass http://api/metrics;
-            allow 127.0.0.1;
-            allow 10.0.0.0/8;
-            deny all;
-        }
-
-        # Documentation
-        location /docs {
-            proxy_pass http://api/docs;
-        }
-
-        location /openapi.json {
-            proxy_pass http://api/openapi.json;
-        }
-
-        # Static files
-        location /static/ {
-            alias /app/static/;
-            expires 1d;
-            add_header Cache-Control "public, immutable";
-        }
-    }
-
-    # SearXNG server (internal use only)
-    server {
-        listen 8081;
-        server_name searxng.internal;
-        
-        location / {
-            proxy_pass http://searxng;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            
-            # Restrict to internal network
-            allow 10.0.0.0/8;
-            allow 172.16.0.0/12;
-            allow 192.168.0.0/16;
-            deny all;
-        }
-    }
-
-    # Flower monitoring (restricted)
-    server {
-        listen 5556;
-        server_name flower.UnSearch.io;
-        
-        location / {
-            proxy_pass http://flower;
-            proxy_set_header Host $host;
-            
-            # Basic auth for Flower
-            auth_basic "Flower Monitoring";
-            auth_basic_user_file /etc/nginx/.htpasswd;
-            
-            # Restrict access
-            allow 127.0.0.1;
-            allow 10.0.0.0/8;
-            deny all;
-        }
-    }
-
-    # HTTPS configuration (uncomment for production)
-    # server {
-    #     listen 443 ssl http2;
-    #     server_name api.UnSearch.io;
-    #     
-    #     ssl_certificate /etc/nginx/ssl/cert.pem;
-    #     ssl_certificate_key /etc/nginx/ssl/key.pem;
-    #     ssl_protocols TLSv1.2 TLSv1.3;
-    #     ssl_ciphers HIGH:!aNULL:!MD5;
-    #     ssl_prefer_server_ciphers on;
-    #     
-    #     # Include other location blocks from above
-    # }
-}
diff --git a/apps/backend/package.json b/apps/backend/package.json
deleted file mode 100644
index 423e6fe..0000000
--- a/apps/backend/package.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-  "name": "@unsearch/backend",
-  "version": "1.0.0",
-  "description": "UnSearch API Backend - FastAPI application",
-  "private": true,
-  "scripts": {
-    "dev": "uvicorn app.main:app --reload --host 0.0.0.0 --port 8000",
-    "build": "echo 'Backend built successfully'",
-    "start": "uvicorn app.main:app --host 0.0.0.0 --port 8000",
-    "test": "pytest",
-    "lint": "flake8 app/ tests/ && black --check app/ tests/",
-    "format": "black app/ tests/ && isort app/ tests/",
-    "migrate": "alembic upgrade head",
-    "worker": "celery -A app.workers.tasks worker --loglevel=info"
-  }
-}
diff --git a/apps/backend/poetry.lock b/apps/backend/poetry.lock
deleted file mode 100644
index 4775dfc..0000000
--- a/apps/backend/poetry.lock
+++ /dev/null
@@ -1,127 +0,0 @@
-# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand.
-
-[[package]]
-name = "fakeredis"
-version = "2.31.1"
-description = "Python implementation of redis API, can be used for testing purposes."
-optional = false
-python-versions = ">=3.7"
-groups = ["dev"]
-files = [
-    {file = "fakeredis-2.31.1-py3-none-any.whl", hash = "sha256:1c0403dedc42bb0038649f016e1a8b56b4b1c69dfb13cf11f870dc51e5c5b4df"},
-    {file = "fakeredis-2.31.1.tar.gz", hash = "sha256:bba58475d6ba3846752d242921c5d3f6dc948066e0ddd054f3a448cd9a1aacad"},
-]
-
-[package.dependencies]
-redis = {version = ">=4.3", markers = "python_version > \"3.8\""}
-sortedcontainers = ">=2,<3"
-
-[package.extras]
-bf = ["pyprobables (>=0.6)"]
-cf = ["pyprobables (>=0.6)"]
-json = ["jsonpath-ng (>=1.6,<2.0)"]
-lua = ["lupa (>=2.1,<3.0)"]
-probabilistic = ["pyprobables (>=0.6)"]
-valkey = ["valkey (>=6) ; python_version >= \"3.8\""]
-
-[[package]]
-name = "greenlet"
-version = "3.2.4"
-description = "Lightweight in-process concurrent programming"
-optional = false
-python-versions = ">=3.9"
-groups = ["main"]
-files = [
-    {file = "greenlet-3.2.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:8c68325b0d0acf8d91dde4e6f930967dd52a5302cd4062932a6b2e7c2969f47c"},
-    {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:94385f101946790ae13da500603491f04a76b6e4c059dab271b3ce2e283b2590"},
-    {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f10fd42b5ee276335863712fa3da6608e93f70629c631bf77145021600abc23c"},
-    {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c8c9e331e58180d0d83c5b7999255721b725913ff6bc6cf39fa2a45841a4fd4b"},
-    {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58b97143c9cc7b86fc458f215bd0932f1757ce649e05b640fea2e79b54cedb31"},
-    {file = "greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d"},
-    {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5"},
-    {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f"},
-    {file = "greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c"},
-    {file = "greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2"},
-    {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246"},
-    {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3"},
-    {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633"},
-    {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079"},
-    {file = "greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8"},
-    {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52"},
-    {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa"},
-    {file = "greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9"},
-    {file = "greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd"},
-    {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb"},
-    {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968"},
-    {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9"},
-    {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6"},
-    {file = "greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0"},
-    {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0"},
-    {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f"},
-    {file = "greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02"},
-    {file = "greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31"},
-    {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945"},
-    {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc"},
-    {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a"},
-    {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504"},
-    {file = "greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671"},
-    {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b"},
-    {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae"},
-    {file = "greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b"},
-    {file = "greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0"},
-    {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f"},
-    {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5"},
-    {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1"},
-    {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735"},
-    {file = "greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337"},
-    {file = "greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01"},
-    {file = "greenlet-3.2.4-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:b6a7c19cf0d2742d0809a4c05975db036fdff50cd294a93632d6a310bf9ac02c"},
-    {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:27890167f55d2387576d1f41d9487ef171849ea0359ce1510ca6e06c8bece11d"},
-    {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:18d9260df2b5fbf41ae5139e1be4e796d99655f023a636cd0e11e6406cca7d58"},
-    {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:671df96c1f23c4a0d4077a325483c1503c96a1b7d9db26592ae770daa41233d4"},
-    {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16458c245a38991aa19676900d48bd1a6f2ce3e16595051a4db9d012154e8433"},
-    {file = "greenlet-3.2.4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9913f1a30e4526f432991f89ae263459b1c64d1608c0d22a5c79c287b3c70df"},
-    {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b90654e092f928f110e0007f572007c9727b5265f7632c2fa7415b4689351594"},
-    {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81701fd84f26330f0d5f4944d4e92e61afe6319dcd9775e39396e39d7c3e5f98"},
-    {file = "greenlet-3.2.4-cp39-cp39-win32.whl", hash = "sha256:65458b409c1ed459ea899e939f0e1cdb14f58dbc803f2f93c5eab5694d32671b"},
-    {file = "greenlet-3.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:d2e685ade4dafd447ede19c31277a224a239a0a1a4eca4e6390efedf20260cfb"},
-    {file = "greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d"},
-]
-
-[package.extras]
-docs = ["Sphinx", "furo"]
-test = ["objgraph", "psutil", "setuptools"]
-
-[[package]]
-name = "redis"
-version = "6.4.0"
-description = "Python client for Redis database and key-value store"
-optional = false
-python-versions = ">=3.9"
-groups = ["dev"]
-files = [
-    {file = "redis-6.4.0-py3-none-any.whl", hash = "sha256:f0544fa9604264e9464cdf4814e7d4830f74b165d52f2a330a760a88dd248b7f"},
-    {file = "redis-6.4.0.tar.gz", hash = "sha256:b01bc7282b8444e28ec36b261df5375183bb47a07eb9c603f284e89cbc5ef010"},
-]
-
-[package.extras]
-hiredis = ["hiredis (>=3.2.0)"]
-jwt = ["pyjwt (>=2.9.0)"]
-ocsp = ["cryptography (>=36.0.1)", "pyopenssl (>=20.0.1)", "requests (>=2.31.0)"]
-
-[[package]]
-name = "sortedcontainers"
-version = "2.4.0"
-description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set"
-optional = false
-python-versions = "*"
-groups = ["dev"]
-files = [
-    {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"},
-    {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"},
-]
-
-[metadata]
-lock-version = "2.1"
-python-versions = ">=3.13,<4.0"
-content-hash = "1ad363eaca1e2cb855d90871c8514472b7a10383eebf3cdaf66c3f5989937a88"
diff --git a/apps/backend/pyproject.toml b/apps/backend/pyproject.toml
deleted file mode 100644
index cd0aa39..0000000
--- a/apps/backend/pyproject.toml
+++ /dev/null
@@ -1,18 +0,0 @@
-[project]
-name = "unsearch-backend"
-version = "0.1.0"
-description = "UnSearch backend API"
-readme = "README.md"
-requires-python = ">=3.13,<4.0"
-dependencies = [
-    "greenlet (>=3.2.4,<4.0.0)"
-]
-
-
-[build-system]
-requires = ["poetry-core>=2.0.0,<3.0.0"]
-build-backend = "poetry.core.masonry.api"
-
-[tool.poetry.group.dev.dependencies]
-fakeredis = "^2.31.1"
-
diff --git a/apps/backend/pytest.ini b/apps/backend/pytest.ini
deleted file mode 100644
index 66cf820..0000000
--- a/apps/backend/pytest.ini
+++ /dev/null
@@ -1,34 +0,0 @@
-[pytest]
-minversion = 7.0
-testpaths = tests
-python_files = test_*.py
-python_classes = Test*
-python_functions = test_*
-addopts = 
-    -ra
-    --strict-markers
-    --ignore=docs
-    --ignore=scripts
-    --ignore=docker
-    --cov=app
-    --cov-report=term-missing:skip-covered
-    --cov-report=html
-    --cov-report=xml
-    --cov-fail-under=80
-    -p no:warnings
-    --asyncio-mode=auto
-
-markers =
-    unit: Unit tests
-    integration: Integration tests
-    performance: Performance tests
-    slow: Slow tests
-    require_searxng: Tests that require SearXNG
-    require_redis: Tests that require Redis
-    require_db: Tests that require database
-
-env =
-    ENVIRONMENT=testing
-    DATABASE_URL=sqlite:///test.db
-    REDIS_URL=redis://localhost:6379/1
-    SEARXNG_URL=http://localhost:8080
diff --git a/apps/backend/railway.json b/apps/backend/railway.json
deleted file mode 100644
index b4cdf5a..0000000
--- a/apps/backend/railway.json
+++ /dev/null
@@ -1,48 +0,0 @@
-{
-  "build": {
-    "builder": "NIXPACKS",
-    "buildCommand": "poetry install --no-root"
-  },
-  "deploy": {
-    "services": {
-      "web": {
-        "startCommand": "ENVIRONMENT=production uvicorn app.main:app --host 0.0.0.0 --port $PORT",
-        "env": {
-          "ENVIRONMENT": "production",
-          "LOG_LEVEL": "INFO",
-          "SEARXNG_URL": "${SEARXNG_URL}",
-          "DATABASE_URL": "${DATABASE_URL}",
-          "REDIS_URL": "${REDIS_URL}",
-          "CELERY_BROKER_URL": "${CELERY_BROKER_URL:-${REDIS_URL}}",
-          "CELERY_RESULT_BACKEND": "${CELERY_RESULT_BACKEND:-${REDIS_URL}}",
-          "ALLOWED_ORIGINS": "[\"*\"]",
-          "CORS_METHODS": "[\"GET\",\"POST\",\"PUT\",\"DELETE\",\"OPTIONS\"]",
-          "CORS_HEADERS": "[\"*\"]"
-        }
-      },
-      "worker": {
-        "startCommand": "celery -A app.workers.tasks worker --loglevel=info --concurrency ${CELERY_WORKER_CONCURRENCY:-4}",
-        "env": {
-          "ENVIRONMENT": "production",
-          "SEARXNG_URL": "${SEARXNG_URL}",
-          "DATABASE_URL": "${DATABASE_URL}",
-          "REDIS_URL": "${REDIS_URL}",
-          "CELERY_BROKER_URL": "${CELERY_BROKER_URL:-${REDIS_URL}}",
-          "CELERY_RESULT_BACKEND": "${CELERY_RESULT_BACKEND:-${REDIS_URL}}"
-        }
-      },
-      "beat": {
-        "startCommand": "celery -A app.workers.tasks beat --loglevel=info",
-        "env": {
-          "ENVIRONMENT": "production",
-          "DATABASE_URL": "${DATABASE_URL}",
-          "REDIS_URL": "${REDIS_URL}",
-          "CELERY_BROKER_URL": "${CELERY_BROKER_URL:-${REDIS_URL}}",
-          "CELERY_RESULT_BACKEND": "${CELERY_RESULT_BACKEND:-${REDIS_URL}}"
-        }
-      }
-    }
-  }
-}
-
-
diff --git a/apps/backend/requirements.txt b/apps/backend/requirements.txt
deleted file mode 100644
index 10150e3..0000000
--- a/apps/backend/requirements.txt
+++ /dev/null
@@ -1,303 +0,0 @@
-#
-# This file is autogenerated by pip-compile with Python 3.13
-# by the following command:
-#
-#    pip-compile requirements.in
-#
-alembic==1.16.5
-    # via -r requirements.in
-amqp==5.3.1
-    # via kombu
-annotated-types==0.7.0
-    # via pydantic
-anyio==4.10.0
-    # via
-    #   httpx
-    #   starlette
-    #   watchfiles
-asyncpg==0.30.0
-    # via -r requirements.in
-bcrypt==4.3.0
-    # via passlib
-beautifulsoup4==4.13.5
-    # via -r requirements.in
-billiard==4.2.1
-    # via celery
-black==25.1.0
-    # via -r requirements.in
-celery==5.5.3
-    # via
-    #   -r requirements.in
-    #   flower
-certifi==2025.8.3
-    # via
-    #   httpcore
-    #   httpx
-cffi==1.17.1
-    # via cryptography
-cfgv==3.4.0
-    # via pre-commit
-chardet==5.2.0
-    # via -r requirements.in
-click==8.2.1
-    # via
-    #   black
-    #   celery
-    #   click-didyoumean
-    #   click-plugins
-    #   click-repl
-    #   nltk
-    #   uvicorn
-click-didyoumean==0.3.1
-    # via celery
-click-plugins==1.1.1.2
-    # via celery
-click-repl==0.3.0
-    # via celery
-coverage[toml]==7.10.6
-    # via pytest-cov
-cryptography==45.0.7
-    # via
-    #   -r requirements.in
-    #   python-jose
-deprecated==1.2.18
-    # via limits
-distlib==0.4.0
-    # via virtualenv
-dnspython==2.7.0
-    # via email-validator
-ecdsa==0.19.1
-    # via python-jose
-email-validator==2.3.0
-    # via -r requirements.in
-environs==14.3.0
-    # via fastapi-cors
-faker==37.6.0
-    # via -r requirements.in
-fastapi==0.116.1
-    # via
-    #   -r requirements.in
-    #   fastapi-cors
-fastapi-cors==0.0.6
-    # via -r requirements.in
-filelock==3.19.1
-    # via virtualenv
-flake8==7.3.0
-    # via -r requirements.in
-flower==2.0.1
-    # via -r requirements.in
-h11==0.16.0
-    # via
-    #   httpcore
-    #   uvicorn
-hiredis==3.2.1
-    # via -r requirements.in
-httpcore==1.0.9
-    # via httpx
-httptools==0.6.4
-    # via uvicorn
-httpx==0.28.1
-    # via
-    #   -r requirements.in
-    #   pytest-httpx
-humanize==4.13.0
-    # via flower
-identify==2.6.13
-    # via pre-commit
-idna==3.10
-    # via
-    #   anyio
-    #   email-validator
-    #   httpx
-iniconfig==2.1.0
-    # via pytest
-isort==6.0.1
-    # via -r requirements.in
-joblib==1.5.2
-    # via nltk
-kombu==5.5.4
-    # via celery
-langdetect==1.0.9
-    # via -r requirements.in
-limits==5.5.0
-    # via slowapi
-lxml==6.0.1
-    # via -r requirements.in
-mako==1.3.10
-    # via alembic
-markupsafe==3.0.2
-    # via mako
-marshmallow==4.0.1
-    # via environs
-mccabe==0.7.0
-    # via flake8
-mypy==1.17.1
-    # via -r requirements.in
-mypy-extensions==1.1.0
-    # via
-    #   black
-    #   mypy
-nltk==3.9.1
-    # via -r requirements.in
-nodeenv==1.9.1
-    # via pre-commit
-orjson==3.11.3
-    # via -r requirements.in
-packaging==25.0
-    # via
-    #   black
-    #   kombu
-    #   limits
-    #   pytest
-passlib[bcrypt]==1.7.4
-    # via -r requirements.in
-pathspec==0.12.1
-    # via
-    #   black
-    #   mypy
-platformdirs==4.4.0
-    # via
-    #   black
-    #   virtualenv
-pluggy==1.6.0
-    # via
-    #   pytest
-    #   pytest-cov
-pre-commit==4.3.0
-    # via -r requirements.in
-prometheus-client==0.22.1
-    # via
-    #   -r requirements.in
-    #   flower
-prompt-toolkit==3.0.52
-    # via click-repl
-psycopg2-binary==2.9.10
-    # via -r requirements.in
-pyasn1==0.6.1
-    # via
-    #   python-jose
-    #   rsa
-pycodestyle==2.14.0
-    # via flake8
-pycparser==2.22
-    # via cffi
-pydantic==2.11.7
-    # via
-    #   -r requirements.in
-    #   fastapi
-    #   pydantic-settings
-pydantic-core==2.33.2
-    # via pydantic
-pydantic-settings==2.10.1
-    # via -r requirements.in
-pyjwt==2.10.1
-    # via -r requirements.in
-stripe==11.4.0
-    # via -r requirements.in
-pyflakes==3.4.0
-    # via flake8
-pygments==2.19.2
-    # via pytest
-pytest==8.4.2
-    # via
-    #   -r requirements.in
-    #   pytest-asyncio
-    #   pytest-cov
-    #   pytest-httpx
-pytest-asyncio==1.1.0
-    # via -r requirements.in
-pytest-cov==6.2.1
-    # via -r requirements.in
-pytest-httpx==0.35.0
-    # via -r requirements.in
-python-dateutil==2.9.0.post0
-    # via celery
-python-dotenv==1.1.1
-    # via
-    #   -r requirements.in
-    #   environs
-    #   pydantic-settings
-    #   uvicorn
-python-jose[cryptography]==3.5.0
-    # via -r requirements.in
-python-json-logger==3.3.0
-    # via -r requirements.in
-python-multipart==0.0.20
-    # via -r requirements.in
-pytz==2025.2
-    # via flower
-pyyaml==6.0.2
-    # via
-    #   -r requirements.in
-    #   pre-commit
-    #   uvicorn
-redis==6.4.0
-    # via -r requirements.in
-regex==2025.9.1
-    # via nltk
-rsa==4.9.1
-    # via python-jose
-six==1.17.0
-    # via
-    #   ecdsa
-    #   langdetect
-    #   python-dateutil
-slowapi==0.1.9
-    # via -r requirements.in
-sniffio==1.3.1
-    # via anyio
-soupsieve==2.8
-    # via beautifulsoup4
-sqlalchemy==2.0.43
-    # via
-    #   -r requirements.in
-    #   alembic
-starlette==0.47.3
-    # via fastapi
-structlog==25.4.0
-    # via -r requirements.in
-tenacity==9.1.2
-    # via -r requirements.in
-tornado==6.5.2
-    # via flower
-tqdm==4.67.1
-    # via nltk
-typing-extensions==4.15.0
-    # via
-    #   alembic
-    #   beautifulsoup4
-    #   fastapi
-    #   limits
-    #   mypy
-    #   pydantic
-    #   pydantic-core
-    #   sqlalchemy
-    #   typing-inspection
-typing-inspection==0.4.1
-    # via
-    #   pydantic
-    #   pydantic-settings
-tzdata==2025.2
-    # via
-    #   faker
-    #   kombu
-uvicorn[standard]==0.35.0
-    # via -r requirements.in
-uvloop==0.21.0
-    # via uvicorn
-vine==5.1.0
-    # via
-    #   amqp
-    #   celery
-    #   kombu
-virtualenv==20.34.0
-    # via pre-commit
-watchfiles==1.1.0
-    # via uvicorn
-wcwidth==0.2.13
-    # via prompt-toolkit
-websockets==15.0.1
-    # via uvicorn
-wrapt==1.17.3
-    # via deprecated
-upstash-redis==1.4.0
diff --git a/apps/backend/scripts/backup.sh b/apps/backend/scripts/backup.sh
deleted file mode 100755
index a44424c..0000000
--- a/apps/backend/scripts/backup.sh
+++ /dev/null
@@ -1,379 +0,0 @@
-#!/bin/bash
-
-# UnSearch API Database Backup Script
-# Performs automated backups of PostgreSQL database and Redis data
-
-set -e  # Exit on error
-set -u  # Exit on undefined variable
-
-# Configuration
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
-BACKUP_DIR="${BACKUP_DIR:-$PROJECT_ROOT/backups}"
-TIMESTAMP=$(date +%Y%m%d_%H%M%S)
-RETENTION_DAYS="${RETENTION_DAYS:-30}"
-
-# Colors for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-# Logging functions
-log_info() {
-    echo -e "${GREEN}[INFO]${NC} $1"
-}
-
-log_warn() {
-    echo -e "${YELLOW}[WARN]${NC} $1"
-}
-
-log_error() {
-    echo -e "${RED}[ERROR]${NC} $1"
-}
-
-# Load environment variables
-if [ -f "$PROJECT_ROOT/.env" ]; then
-    export $(cat "$PROJECT_ROOT/.env" | grep -v '^#' | xargs)
-else
-    log_warn ".env file not found. Using default values."
-fi
-
-# Database configuration
-DB_HOST="${DB_HOST:-localhost}"
-DB_PORT="${DB_PORT:-5432}"
-DB_NAME="${DB_NAME:-UnSearch}"
-DB_USER="${DB_USER:-UnSearch}"
-DB_PASSWORD="${DB_PASSWORD:-UnSearch123}"
-
-# Redis configuration
-REDIS_HOST="${REDIS_HOST:-localhost}"
-REDIS_PORT="${REDIS_PORT:-6379}"
-
-# S3 configuration (optional)
-S3_BUCKET="${S3_BUCKET:-}"
-S3_REGION="${S3_REGION:-us-east-1}"
-
-# Function to check if command exists
-command_exists() {
-    command -v "$1" >/dev/null 2>&1
-}
-
-# Function to create backup directory
-create_backup_dir() {
-    if [ ! -d "$BACKUP_DIR" ]; then
-        mkdir -p "$BACKUP_DIR"
-        log_info "Created backup directory: $BACKUP_DIR"
-    fi
-}
-
-# Function to backup PostgreSQL database
-backup_postgres() {
-    log_info "Starting PostgreSQL backup..."
-    
-    local backup_file="$BACKUP_DIR/postgres_${DB_NAME}_${TIMESTAMP}.sql.gz"
-    
-    # Check if PostgreSQL is accessible
-    PGPASSWORD=$DB_PASSWORD pg_isready -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME > /dev/null 2>&1
-    if [ $? -ne 0 ]; then
-        log_error "Cannot connect to PostgreSQL database"
-        return 1
-    fi
-    
-    # Perform backup
-    PGPASSWORD=$DB_PASSWORD pg_dump \
-        -h $DB_HOST \
-        -p $DB_PORT \
-        -U $DB_USER \
-        -d $DB_NAME \
-        --verbose \
-        --clean \
-        --if-exists \
-        --no-owner \
-        --no-acl \
-        --format=plain \
-        --encoding=UTF8 | gzip -9 > "$backup_file"
-    
-    if [ $? -eq 0 ]; then
-        local size=$(du -h "$backup_file" | cut -f1)
-        log_info "PostgreSQL backup completed: $backup_file (Size: $size)"
-        echo "$backup_file"
-    else
-        log_error "PostgreSQL backup failed"
-        return 1
-    fi
-}
-
-# Function to backup Redis data
-backup_redis() {
-    log_info "Starting Redis backup..."
-    
-    local backup_file="$BACKUP_DIR/redis_${TIMESTAMP}.rdb"
-    
-    # Check if Redis is accessible
-    redis-cli -h $REDIS_HOST -p $REDIS_PORT ping > /dev/null 2>&1
-    if [ $? -ne 0 ]; then
-        log_error "Cannot connect to Redis"
-        return 1
-    fi
-    
-    # Trigger Redis save
-    redis-cli -h $REDIS_HOST -p $REDIS_PORT BGSAVE
-    
-    # Wait for save to complete
-    log_info "Waiting for Redis save to complete..."
-    while [ "$(redis-cli -h $REDIS_HOST -p $REDIS_PORT LASTSAVE)" = "$(redis-cli -h $REDIS_HOST -p $REDIS_PORT LASTSAVE)" ]; do
-        sleep 1
-    done
-    
-    # Copy the dump file
-    if [ -f "/var/lib/redis/dump.rdb" ]; then
-        cp /var/lib/redis/dump.rdb "$backup_file"
-    elif [ -f "./dump.rdb" ]; then
-        cp ./dump.rdb "$backup_file"
-    else
-        # Try to get it from Docker container
-        docker cp UnSearch-redis:/data/dump.rdb "$backup_file" 2>/dev/null || {
-            log_warn "Could not find Redis dump file"
-            return 1
-        }
-    fi
-    
-    # Compress the backup
-    gzip -9 "$backup_file"
-    backup_file="${backup_file}.gz"
-    
-    if [ -f "$backup_file" ]; then
-        local size=$(du -h "$backup_file" | cut -f1)
-        log_info "Redis backup completed: $backup_file (Size: $size)"
-        echo "$backup_file"
-    else
-        log_error "Redis backup failed"
-        return 1
-    fi
-}
-
-# Function to backup application files
-backup_application() {
-    log_info "Starting application files backup..."
-    
-    local backup_file="$BACKUP_DIR/app_files_${TIMESTAMP}.tar.gz"
-    
-    # Create list of important files/directories to backup
-    local files_to_backup=(
-        ".env"
-        "alembic"
-        "searxng/settings.yml"
-        "nginx/nginx.conf"
-        "scripts"
-    )
-    
-    # Change to project root
-    cd "$PROJECT_ROOT"
-    
-    # Create tar archive
-    tar -czf "$backup_file" "${files_to_backup[@]}" 2>/dev/null || {
-        log_warn "Some files might not exist, continuing..."
-    }
-    
-    if [ -f "$backup_file" ]; then
-        local size=$(du -h "$backup_file" | cut -f1)
-        log_info "Application files backup completed: $backup_file (Size: $size)"
-        echo "$backup_file"
-    else
-        log_error "Application files backup failed"
-        return 1
-    fi
-}
-
-# Function to upload to S3
-upload_to_s3() {
-    local file=$1
-    
-    if [ -z "$S3_BUCKET" ]; then
-        log_info "S3 backup not configured, skipping upload"
-        return 0
-    fi
-    
-    if ! command_exists aws; then
-        log_warn "AWS CLI not installed, skipping S3 upload"
-        return 0
-    fi
-    
-    log_info "Uploading to S3: $file"
-    
-    local filename=$(basename "$file")
-    local s3_path="s3://$S3_BUCKET/backups/UnSearch/$(date +%Y/%m/%d)/$filename"
-    
-    aws s3 cp "$file" "$s3_path" --region "$S3_REGION"
-    
-    if [ $? -eq 0 ]; then
-        log_info "Successfully uploaded to S3: $s3_path"
-        return 0
-    else
-        log_error "Failed to upload to S3"
-        return 1
-    fi
-}
-
-# Function to clean old backups
-cleanup_old_backups() {
-    log_info "Cleaning up old backups (older than $RETENTION_DAYS days)..."
-    
-    find "$BACKUP_DIR" -type f -name "*.gz" -mtime +$RETENTION_DAYS -delete
-    find "$BACKUP_DIR" -type f -name "*.sql" -mtime +$RETENTION_DAYS -delete
-    find "$BACKUP_DIR" -type f -name "*.rdb" -mtime +$RETENTION_DAYS -delete
-    find "$BACKUP_DIR" -type f -name "*.tar" -mtime +$RETENTION_DAYS -delete
-    
-    log_info "Cleanup completed"
-}
-
-# Function to create backup report
-create_backup_report() {
-    local report_file="$BACKUP_DIR/backup_report_${TIMESTAMP}.txt"
-    
-    cat > "$report_file" << EOF
-========================================
-UnSearch API Backup Report
-========================================
-Date: $(date)
-Hostname: $(hostname)
-User: $(whoami)
-
-Backup Details:
-----------------------------------------
-EOF
-    
-    for file in "$@"; do
-        if [ -f "$file" ]; then
-            echo "- $(basename "$file"): $(du -h "$file" | cut -f1)" >> "$report_file"
-        fi
-    done
-    
-    cat >> "$report_file" << EOF
-
-Backup Location: $BACKUP_DIR
-Retention Policy: $RETENTION_DAYS days
-S3 Bucket: ${S3_BUCKET:-Not configured}
-
-========================================
-EOF
-    
-    log_info "Backup report created: $report_file"
-    cat "$report_file"
-}
-
-# Function to send notification (optional)
-send_notification() {
-    local status=$1
-    local message=$2
-    
-    # Slack webhook notification (if configured)
-    if [ ! -z "${SLACK_WEBHOOK_URL:-}" ]; then
-        local color="good"
-        if [ "$status" != "success" ]; then
-            color="danger"
-        fi
-        
-        curl -X POST "$SLACK_WEBHOOK_URL" \
-            -H 'Content-Type: application/json' \
-            -d "{
-                \"attachments\": [{
-                    \"color\": \"$color\",
-                    \"title\": \"UnSearch Backup\",
-                    \"text\": \"$message\",
-                    \"footer\": \"Backup System\",
-                    \"ts\": $(date +%s)
-                }]
-            }" 2>/dev/null
-    fi
-    
-    # Email notification (if configured)
-    if [ ! -z "${BACKUP_EMAIL:-}" ] && command_exists mail; then
-        echo "$message" | mail -s "UnSearch Backup - $status" "$BACKUP_EMAIL"
-    fi
-}
-
-# Main backup process
-main() {
-    log_info "========================================="
-    log_info "Starting UnSearch API Backup"
-    log_info "Timestamp: $TIMESTAMP"
-    log_info "========================================="
-    
-    # Check required commands
-    for cmd in pg_dump redis-cli tar gzip; do
-        if ! command_exists $cmd; then
-            log_error "Required command not found: $cmd"
-            exit 1
-        fi
-    done
-    
-    # Create backup directory
-    create_backup_dir
-    
-    # Array to store backup files
-    declare -a backup_files
-    
-    # Perform backups
-    if pg_backup=$(backup_postgres); then
-        backup_files+=("$pg_backup")
-        [ ! -z "$S3_BUCKET" ] && upload_to_s3 "$pg_backup"
-    else
-        log_error "PostgreSQL backup failed, continuing..."
-    fi
-    
-    if redis_backup=$(backup_redis); then
-        backup_files+=("$redis_backup")
-        [ ! -z "$S3_BUCKET" ] && upload_to_s3 "$redis_backup"
-    else
-        log_error "Redis backup failed, continuing..."
-    fi
-    
-    if app_backup=$(backup_application); then
-        backup_files+=("$app_backup")
-        [ ! -z "$S3_BUCKET" ] && upload_to_s3 "$app_backup"
-    else
-        log_error "Application backup failed, continuing..."
-    fi
-    
-    # Clean old backups
-    cleanup_old_backups
-    
-    # Create report
-    create_backup_report "${backup_files[@]}"
-    
-    # Send notification
-    if [ ${#backup_files[@]} -gt 0 ]; then
-        send_notification "success" "Backup completed successfully. ${#backup_files[@]} files backed up."
-        log_info "========================================="
-        log_info "Backup completed successfully!"
-        log_info "========================================="
-        exit 0
-    else
-        send_notification "failure" "Backup failed. No files were backed up."
-        log_error "========================================="
-        log_error "Backup failed!"
-        log_error "========================================="
-        exit 1
-    fi
-}
-
-# Handle script arguments
-case "${1:-}" in
-    postgres)
-        create_backup_dir
-        backup_postgres
-        ;;
-    redis)
-        create_backup_dir
-        backup_redis
-        ;;
-    app)
-        create_backup_dir
-        backup_application
-        ;;
-    *)
-        main
-        ;;
-esac
diff --git a/apps/backend/scripts/deploy.sh b/apps/backend/scripts/deploy.sh
deleted file mode 100755
index c4866ed..0000000
--- a/apps/backend/scripts/deploy.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-"""
-Deployment script for UnSearch API.
-"""
-
-set -e
-
-# Configuration
-APP_NAME="UnSearch-api"
-DOCKER_IMAGE="UnSearch/api"
-DOCKER_TAG="${1:-latest}"
-
-echo "🚀 Deploying UnSearch API..."
-echo "📦 Image: $DOCKER_IMAGE:$DOCKER_TAG"
-
-# Check if Docker is available
-if ! command -v docker &> /dev/null; then
-    echo "❌ Docker not found. Please install Docker first."
-    exit 1
-fi
-
-# Check if we're in the right directory
-if [ ! -f "Dockerfile" ]; then
-    echo "❌ Dockerfile not found. Please run from project root."
-    exit 1
-fi
-
-# Build Docker image
-echo "🔨 Building Docker image..."
-docker build -t "$DOCKER_IMAGE:$DOCKER_TAG" .
-
-# Tag as latest if not already latest
-if [ "$DOCKER_TAG" != "latest" ]; then
-    docker tag "$DOCKER_IMAGE:$DOCKER_TAG" "$DOCKER_IMAGE:latest"
-fi
-
-echo "✅ Docker image built successfully"
-
-# Check deployment type
-case "${2:-local}" in
-    "local")
-        echo "🏠 Deploying locally with Docker Compose..."
-        
-        # Stop existing containers
-        docker-compose down
-        
-        # Start services
-        docker-compose up -d
-        
-        # Wait for services to be ready
-        echo "⏳ Waiting for services to start..."
-        sleep 10
-        
-        # Check health
-        echo "🏥 Checking service health..."
-        if curl -f http://localhost:8000/health > /dev/null 2>&1; then
-            echo "✅ API is healthy"
-        else
-            echo "⚠️ API health check failed"
-        fi
-        
-        echo "🎉 Local deployment complete!"
-        echo "📖 API Documentation: http://localhost:8000/docs"
-        echo "📊 Metrics: http://localhost:8000/metrics"
-        ;;
-        
-    "staging")
-        echo "🎭 Deploying to staging..."
-        
-        # Push to registry (configure your registry)
-        # docker push "$DOCKER_IMAGE:$DOCKER_TAG"
-        
-        # Deploy to staging environment
-        # kubectl apply -f k8s/staging/
-        
-        echo "⚠️ Staging deployment not fully implemented"
-        echo "💡 Configure your staging environment in this script"
-        ;;
-        
-    "production")
-        echo "🏭 Deploying to production..."
-        
-        # Safety check
-        read -p "⚠️ Are you sure you want to deploy to production? (yes/no): " confirm
-        if [ "$confirm" != "yes" ]; then
-            echo "❌ Production deployment cancelled"
-            exit 1
-        fi
-        
-        # Push to registry
-        # docker push "$DOCKER_IMAGE:$DOCKER_TAG"
-        
-        # Deploy to production environment
-        # kubectl apply -f k8s/production/
-        
-        echo "⚠️ Production deployment not fully implemented"
-        echo "💡 Configure your production environment in this script"
-        ;;
-        
-    *)
-        echo "❌ Unknown deployment target: $2"
-        echo "📖 Usage: $0 [tag] [local|staging|production]"
-        exit 1
-        ;;
-esac
-
-echo "🎉 Deployment script completed!"
\ No newline at end of file
diff --git a/apps/backend/scripts/health_check.sh b/apps/backend/scripts/health_check.sh
deleted file mode 100755
index 1979d37..0000000
--- a/apps/backend/scripts/health_check.sh
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/bin/bash
-
-# Health check script for UnSearch API
-# Can be used for monitoring and automated health checks
-
-set -e
-
-# Configuration
-API_URL=${API_URL:-"http://localhost:8000"}
-TIMEOUT=${TIMEOUT:-10}
-
-# Colors
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m'
-
-echo "UnSearch API Health Check"
-echo "============================="
-echo ""
-
-# Function to check service
-check_service() {
-    local name=$1
-    local url=$2
-    local expected=$3
-    
-    printf "Checking %-20s" "$name..."
-    
-    response=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout $TIMEOUT "$url" || echo "000")
-    
-    if [ "$response" = "$expected" ]; then
-        echo -e "${GREEN}✓ OK${NC}"
-        return 0
-    else
-        echo -e "${RED}✗ FAILED (HTTP $response)${NC}"
-        return 1
-    fi
-}
-
-# Check basic health
-check_service "Basic Health" "$API_URL/health" "200"
-
-# Check detailed health
-echo ""
-echo "Detailed Health Check:"
-health_response=$(curl -s --connect-timeout $TIMEOUT "$API_URL/api/v1/search/health" || echo "{}")
-
-if command -v jq >/dev/null 2>&1; then
-    status=$(echo "$health_response" | jq -r '.status' 2>/dev/null || echo "unknown")
-    
-    if [ "$status" = "healthy" ]; then
-        echo -e "Overall Status: ${GREEN}$status${NC}"
-    elif [ "$status" = "degraded" ]; then
-        echo -e "Overall Status: ${YELLOW}$status${NC}"
-    else
-        echo -e "Overall Status: ${RED}$status${NC}"
-    fi
-    
-    echo ""
-    echo "Service Status:"
-    echo "$health_response" | jq -r '.services | to_entries[] | "- \(.key): \(.value.status) (latency: \(.value.latency_ms)ms)"' 2>/dev/null || echo "Failed to parse service status"
-else
-    echo "$health_response"
-fi
-
-# Check API docs
-echo ""
-check_service "API Documentation" "$API_URL/docs" "200"
-
-# Check metrics endpoint
-check_service "Metrics Endpoint" "$API_URL/metrics" "200"
-
-# Performance check
-echo ""
-echo "Performance Check:"
-start_time=$(date +%s%N)
-response=$(curl -s -X POST "$API_URL/api/v1/search" \
-    -H "Content-Type: application/json" \
-    -H "X-API-Key: test-key" \
-    -d '{"query":"test","engines":["google"],"max_results":1,"scrape_content":false}' \
-    --connect-timeout $TIMEOUT || echo "{}")
-end_time=$(date +%s%N)
-
-if command -v jq >/dev/null 2>&1 && [ -n "$response" ]; then
-    request_id=$(echo "$response" | jq -r '.request_id' 2>/dev/null || echo "N/A")
-    processing_time=$(echo "$response" | jq -r '.processing_time_ms' 2>/dev/null || echo "N/A")
-    
-    total_time=$((($end_time - $start_time) / 1000000))
-    
-    echo "- Request ID: $request_id"
-    echo "- Server Processing Time: ${processing_time}ms"
-    echo "- Total Round Trip Time: ${total_time}ms"
-fi
-
-echo ""
-echo "Health check completed."
diff --git a/apps/backend/scripts/monitor.sh b/apps/backend/scripts/monitor.sh
deleted file mode 100755
index 198be8b..0000000
--- a/apps/backend/scripts/monitor.sh
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/bin/bash
-"""
-Monitoring and health check script for UnSearch API.
-"""
-
-set -e
-
-API_URL="${1:-http://localhost:8000}"
-CHECK_INTERVAL="${2:-30}"
-
-echo "🔍 Starting UnSearch API monitoring..."
-echo "🌐 API URL: $API_URL"
-echo "⏱️ Check interval: ${CHECK_INTERVAL}s"
-
-# Colors for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-# Function to check API health
-check_health() {
-    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
-    
-    # Check main health endpoint
-    if response=$(curl -s -w "HTTPSTATUS:%{http_code};TIME:%{time_total}" "$API_URL/health" 2>/dev/null); then
-        http_code=$(echo $response | grep -o "HTTPSTATUS:[0-9]*" | cut -d: -f2)
-        time_total=$(echo $response | grep -o "TIME:[0-9.]*" | cut -d: -f2)
-        body=$(echo $response | sed -E 's/HTTPSTATUS:[0-9]*;TIME:[0-9.]*$//')
-        
-        if [ "$http_code" = "200" ]; then
-            status=$(echo $body | grep -o '"status":"[^"]*' | cut -d'"' -f4)
-            if [ "$status" = "healthy" ]; then
-                echo -e "${GREEN}✅ [$timestamp] API healthy - Response time: ${time_total}s${NC}"
-                return 0
-            else
-                echo -e "${YELLOW}⚠️ [$timestamp] API degraded - Status: $status${NC}"
-                return 1
-            fi
-        else
-            echo -e "${RED}❌ [$timestamp] API unhealthy - HTTP $http_code${NC}"
-            return 2
-        fi
-    else
-        echo -e "${RED}❌ [$timestamp] API unreachable${NC}"
-        return 3
-    fi
-}
-
-# Function to check metrics endpoint
-check_metrics() {
-    if curl -s "$API_URL/metrics" > /dev/null 2>&1; then
-        echo -e "${GREEN}📊 Metrics endpoint accessible${NC}"
-    else
-        echo -e "${YELLOW}⚠️ Metrics endpoint not accessible${NC}"
-    fi
-}
-
-# Function to check API endpoints
-check_endpoints() {
-    echo "🔍 Checking API endpoints..."
-    
-    # Check docs
-    if curl -s "$API_URL/docs" > /dev/null 2>&1; then
-        echo -e "${GREEN}📚 Documentation accessible${NC}"
-    else
-        echo -e "${YELLOW}⚠️ Documentation not accessible${NC}"
-    fi
-    
-    # Check OpenAPI schema
-    if curl -s "$API_URL/openapi.json" > /dev/null 2>&1; then
-        echo -e "${GREEN}📋 OpenAPI schema accessible${NC}"
-    else
-        echo -e "${YELLOW}⚠️ OpenAPI schema not accessible${NC}"
-    fi
-}
-
-# Function to show system resources
-show_resources() {
-    if command -v docker &> /dev/null; then
-        echo "🐳 Docker container status:"
-        docker ps --filter "name=UnSearch" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
-        
-        echo ""
-        echo "💾 Container resource usage:"
-        docker stats --no-stream --filter "name=UnSearch" --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}"
-    fi
-}
-
-# Function for continuous monitoring
-continuous_monitor() {
-    echo "🔄 Starting continuous monitoring (Press Ctrl+C to stop)..."
-    
-    while true; do
-        check_health
-        sleep $CHECK_INTERVAL
-    done
-}
-
-# Function for single check
-single_check() {
-    echo "🏥 Performing health check..."
-    check_health
-    
-    echo ""
-    check_metrics
-    
-    echo ""
-    check_endpoints
-    
-    echo ""
-    show_resources
-    
-    echo ""
-    echo "🔗 Useful URLs:"
-    echo "  • API Health: $API_URL/health"
-    echo "  • Documentation: $API_URL/docs"
-    echo "  • Metrics: $API_URL/metrics"
-    echo "  • OpenAPI Schema: $API_URL/openapi.json"
-}
-
-# Handle script arguments
-case "${3:-check}" in
-    "monitor")
-        continuous_monitor
-        ;;
-    "check"|*)
-        single_check
-        ;;
-esac
diff --git a/apps/backend/scripts/restore.sh b/apps/backend/scripts/restore.sh
deleted file mode 100755
index fff6ab8..0000000
--- a/apps/backend/scripts/restore.sh
+++ /dev/null
@@ -1,521 +0,0 @@
-#!/bin/bash
-
-# UnSearch API Database Restore Script
-# Restores PostgreSQL database and Redis data from backups
-
-set -e  # Exit on error
-set -u  # Exit on undefined variable
-
-# Configuration
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
-BACKUP_DIR="${BACKUP_DIR:-$PROJECT_ROOT/backups}"
-
-# Colors for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-BLUE='\033[0;34m'
-NC='\033[0m' # No Color
-
-# Logging functions
-log_info() {
-    echo -e "${GREEN}[INFO]${NC} $1"
-}
-
-log_warn() {
-    echo -e "${YELLOW}[WARN]${NC} $1"
-}
-
-log_error() {
-    echo -e "${RED}[ERROR]${NC} $1"
-}
-
-log_prompt() {
-    echo -e "${BLUE}[PROMPT]${NC} $1"
-}
-
-# Load environment variables
-if [ -f "$PROJECT_ROOT/.env" ]; then
-    export $(cat "$PROJECT_ROOT/.env" | grep -v '^#' | xargs)
-else
-    log_warn ".env file not found. Using default values."
-fi
-
-# Database configuration
-DB_HOST="${DB_HOST:-localhost}"
-DB_PORT="${DB_PORT:-5432}"
-DB_NAME="${DB_NAME:-UnSearch}"
-DB_USER="${DB_USER:-UnSearch}"
-DB_PASSWORD="${DB_PASSWORD:-UnSearch123}"
-
-# Redis configuration
-REDIS_HOST="${REDIS_HOST:-localhost}"
-REDIS_PORT="${REDIS_PORT:-6379}"
-
-# S3 configuration (optional)
-S3_BUCKET="${S3_BUCKET:-}"
-S3_REGION="${S3_REGION:-us-east-1}"
-
-# Function to check if command exists
-command_exists() {
-    command -v "$1" >/dev/null 2>&1
-}
-
-# Function to list available backups
-list_backups() {
-    local backup_type=$1
-    
-    log_info "Available $backup_type backups:"
-    
-    case $backup_type in
-        postgres)
-            find "$BACKUP_DIR" -name "postgres_*.sql.gz" -type f | sort -r | head -20
-            ;;
-        redis)
-            find "$BACKUP_DIR" -name "redis_*.rdb.gz" -type f | sort -r | head -20
-            ;;
-        app)
-            find "$BACKUP_DIR" -name "app_files_*.tar.gz" -type f | sort -r | head -20
-            ;;
-        *)
-            find "$BACKUP_DIR" -name "*.gz" -type f | sort -r | head -20
-            ;;
-    esac
-}
-
-# Function to download from S3
-download_from_s3() {
-    local s3_path=$1
-    local local_file=$2
-    
-    if [ -z "$S3_BUCKET" ]; then
-        log_error "S3 bucket not configured"
-        return 1
-    fi
-    
-    if ! command_exists aws; then
-        log_error "AWS CLI not installed"
-        return 1
-    fi
-    
-    log_info "Downloading from S3: $s3_path"
-    
-    aws s3 cp "$s3_path" "$local_file" --region "$S3_REGION"
-    
-    if [ $? -eq 0 ]; then
-        log_info "Successfully downloaded: $local_file"
-        return 0
-    else
-        log_error "Failed to download from S3"
-        return 1
-    fi
-}
-
-# Function to confirm action
-confirm_action() {
-    local message=$1
-    local response
-    
-    log_prompt "$message (yes/no): "
-    read -r response
-    
-    case $response in
-        [yY][eE][sS]|[yY])
-            return 0
-            ;;
-        *)
-            return 1
-            ;;
-    esac
-}
-
-# Function to restore PostgreSQL database
-restore_postgres() {
-    local backup_file=$1
-    
-    if [ ! -f "$backup_file" ]; then
-        log_error "Backup file not found: $backup_file"
-        return 1
-    fi
-    
-    log_info "Restoring PostgreSQL from: $backup_file"
-    
-    # Check if PostgreSQL is accessible
-    PGPASSWORD=$DB_PASSWORD pg_isready -h $DB_HOST -p $DB_PORT -U $DB_USER > /dev/null 2>&1
-    if [ $? -ne 0 ]; then
-        log_error "Cannot connect to PostgreSQL database"
-        return 1
-    fi
-    
-    # Warn about data loss
-    log_warn "⚠️  This will REPLACE ALL DATA in database: $DB_NAME"
-    if ! confirm_action "Are you sure you want to continue?"; then
-        log_info "Restore cancelled"
-        return 0
-    fi
-    
-    # Create backup of current database before restore
-    log_info "Creating backup of current database..."
-    local pre_restore_backup="$BACKUP_DIR/pre_restore_$(date +%Y%m%d_%H%M%S).sql.gz"
-    PGPASSWORD=$DB_PASSWORD pg_dump \
-        -h $DB_HOST \
-        -p $DB_PORT \
-        -U $DB_USER \
-        -d $DB_NAME \
-        --no-owner \
-        --no-acl | gzip -9 > "$pre_restore_backup"
-    log_info "Current database backed up to: $pre_restore_backup"
-    
-    # Terminate active connections
-    log_info "Terminating active database connections..."
-    PGPASSWORD=$DB_PASSWORD psql \
-        -h $DB_HOST \
-        -p $DB_PORT \
-        -U $DB_USER \
-        -d postgres \
-        -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$DB_NAME' AND pid <> pg_backend_pid();" \
-        > /dev/null 2>&1
-    
-    # Drop and recreate database
-    log_info "Recreating database..."
-    PGPASSWORD=$DB_PASSWORD psql \
-        -h $DB_HOST \
-        -p $DB_PORT \
-        -U $DB_USER \
-        -d postgres \
-        -c "DROP DATABASE IF EXISTS $DB_NAME; CREATE DATABASE $DB_NAME;" \
-        > /dev/null 2>&1
-    
-    # Restore from backup
-    log_info "Restoring database..."
-    gunzip -c "$backup_file" | PGPASSWORD=$DB_PASSWORD psql \
-        -h $DB_HOST \
-        -p $DB_PORT \
-        -U $DB_USER \
-        -d $DB_NAME \
-        --set ON_ERROR_STOP=on
-    
-    if [ $? -eq 0 ]; then
-        log_info "PostgreSQL restore completed successfully"
-        
-        # Run migrations
-        log_info "Running database migrations..."
-        cd "$PROJECT_ROOT"
-        alembic upgrade head
-        
-        return 0
-    else
-        log_error "PostgreSQL restore failed"
-        log_info "Attempting to restore from pre-restore backup..."
-        gunzip -c "$pre_restore_backup" | PGPASSWORD=$DB_PASSWORD psql \
-            -h $DB_HOST \
-            -p $DB_PORT \
-            -U $DB_USER \
-            -d $DB_NAME
-        return 1
-    fi
-}
-
-# Function to restore Redis data
-restore_redis() {
-    local backup_file=$1
-    
-    if [ ! -f "$backup_file" ]; then
-        log_error "Backup file not found: $backup_file"
-        return 1
-    fi
-    
-    log_info "Restoring Redis from: $backup_file"
-    
-    # Check if Redis is accessible
-    redis-cli -h $REDIS_HOST -p $REDIS_PORT ping > /dev/null 2>&1
-    if [ $? -ne 0 ]; then
-        log_error "Cannot connect to Redis"
-        return 1
-    fi
-    
-    # Warn about data loss
-    log_warn "⚠️  This will REPLACE ALL DATA in Redis"
-    if ! confirm_action "Are you sure you want to continue?"; then
-        log_info "Restore cancelled"
-        return 0
-    fi
-    
-    # Create backup of current Redis data
-    log_info "Creating backup of current Redis data..."
-    redis-cli -h $REDIS_HOST -p $REDIS_PORT BGSAVE
-    sleep 2  # Wait for save to start
-    
-    # Extract backup file
-    local temp_file="/tmp/redis_restore_$(date +%s).rdb"
-    gunzip -c "$backup_file" > "$temp_file"
-    
-    # Stop Redis (if using Docker)
-    if command_exists docker && docker ps | grep -q UnSearch-redis; then
-        log_info "Stopping Redis container..."
-        docker stop UnSearch-redis
-        
-        # Copy dump file
-        docker cp "$temp_file" UnSearch-redis:/data/dump.rdb
-        
-        # Start Redis
-        log_info "Starting Redis container..."
-        docker start UnSearch-redis
-    else
-        # For local Redis
-        log_info "Stopping Redis service..."
-        sudo systemctl stop redis || sudo service redis stop
-        
-        # Copy dump file
-        sudo cp "$temp_file" /var/lib/redis/dump.rdb
-        sudo chown redis:redis /var/lib/redis/dump.rdb
-        
-        # Start Redis
-        log_info "Starting Redis service..."
-        sudo systemctl start redis || sudo service redis start
-    fi
-    
-    # Clean up temp file
-    rm -f "$temp_file"
-    
-    # Verify restoration
-    sleep 2
-    redis-cli -h $REDIS_HOST -p $REDIS_PORT ping > /dev/null 2>&1
-    if [ $? -eq 0 ]; then
-        local keys_count=$(redis-cli -h $REDIS_HOST -p $REDIS_PORT DBSIZE | cut -d' ' -f2)
-        log_info "Redis restore completed successfully. Keys in database: $keys_count"
-        return 0
-    else
-        log_error "Redis restore failed"
-        return 1
-    fi
-}
-
-# Function to restore application files
-restore_application() {
-    local backup_file=$1
-    
-    if [ ! -f "$backup_file" ]; then
-        log_error "Backup file not found: $backup_file"
-        return 1
-    fi
-    
-    log_info "Restoring application files from: $backup_file"
-    
-    # Warn about overwriting files
-    log_warn "⚠️  This will overwrite configuration files"
-    if ! confirm_action "Are you sure you want to continue?"; then
-        log_info "Restore cancelled"
-        return 0
-    fi
-    
-    # Create backup of current files
-    log_info "Creating backup of current configuration..."
-    local pre_restore_backup="$BACKUP_DIR/app_pre_restore_$(date +%Y%m%d_%H%M%S).tar.gz"
-    cd "$PROJECT_ROOT"
-    tar -czf "$pre_restore_backup" .env alembic searxng nginx scripts 2>/dev/null || true
-    log_info "Current configuration backed up to: $pre_restore_backup"
-    
-    # Extract backup
-    log_info "Extracting application files..."
-    tar -xzf "$backup_file" -C "$PROJECT_ROOT"
-    
-    if [ $? -eq 0 ]; then
-        log_info "Application files restore completed successfully"
-        
-        # Reload services if running
-        if command_exists docker && docker ps | grep -q UnSearch-api; then
-            log_info "Reloading services..."
-            docker compose restart api nginx
-        fi
-        
-        return 0
-    else
-        log_error "Application files restore failed"
-        return 1
-    fi
-}
-
-# Function to perform full restore
-full_restore() {
-    log_info "Performing full system restore"
-    
-    # Find latest backups
-    local latest_postgres=$(find "$BACKUP_DIR" -name "postgres_*.sql.gz" -type f | sort -r | head -1)
-    local latest_redis=$(find "$BACKUP_DIR" -name "redis_*.rdb.gz" -type f | sort -r | head -1)
-    local latest_app=$(find "$BACKUP_DIR" -name "app_files_*.tar.gz" -type f | sort -r | head -1)
-    
-    if [ -z "$latest_postgres" ] && [ -z "$latest_redis" ] && [ -z "$latest_app" ]; then
-        log_error "No backup files found in $BACKUP_DIR"
-        return 1
-    fi
-    
-    log_info "Found backups:"
-    [ ! -z "$latest_postgres" ] && log_info "  PostgreSQL: $(basename $latest_postgres)"
-    [ ! -z "$latest_redis" ] && log_info "  Redis: $(basename $latest_redis)"
-    [ ! -z "$latest_app" ] && log_info "  Application: $(basename $latest_app)"
-    
-    if ! confirm_action "Restore from these backups?"; then
-        log_info "Restore cancelled"
-        return 0
-    fi
-    
-    # Stop services
-    log_info "Stopping services..."
-    if command_exists docker && docker compose ps | grep -q Up; then
-        docker compose stop
-    fi
-    
-    # Restore each component
-    local restore_success=true
-    
-    if [ ! -z "$latest_app" ]; then
-        restore_application "$latest_app" || restore_success=false
-    fi
-    
-    if [ ! -z "$latest_postgres" ]; then
-        restore_postgres "$latest_postgres" || restore_success=false
-    fi
-    
-    if [ ! -z "$latest_redis" ]; then
-        restore_redis "$latest_redis" || restore_success=false
-    fi
-    
-    # Start services
-    log_info "Starting services..."
-    if command_exists docker; then
-        docker compose up -d
-    fi
-    
-    if $restore_success; then
-        log_info "Full restore completed successfully"
-        return 0
-    else
-        log_error "Some components failed to restore"
-        return 1
-    fi
-}
-
-# Interactive restore menu
-interactive_restore() {
-    while true; do
-        echo
-        log_info "========================================="
-        log_info "UnSearch API Restore Menu"
-        log_info "========================================="
-        echo "1) Restore PostgreSQL database"
-        echo "2) Restore Redis data"
-        echo "3) Restore application files"
-        echo "4) Full system restore (all components)"
-        echo "5) List available backups"
-        echo "6) Download backup from S3"
-        echo "0) Exit"
-        echo
-        log_prompt "Enter your choice [0-6]: "
-        
-        read -r choice
-        
-        case $choice in
-            1)
-                list_backups postgres
-                echo
-                log_prompt "Enter the full path to PostgreSQL backup file: "
-                read -r backup_file
-                restore_postgres "$backup_file"
-                ;;
-            2)
-                list_backups redis
-                echo
-                log_prompt "Enter the full path to Redis backup file: "
-                read -r backup_file
-                restore_redis "$backup_file"
-                ;;
-            3)
-                list_backups app
-                echo
-                log_prompt "Enter the full path to application backup file: "
-                read -r backup_file
-                restore_application "$backup_file"
-                ;;
-            4)
-                full_restore
-                ;;
-            5)
-                log_info "All available backups:"
-                list_backups all
-                ;;
-            6)
-                log_prompt "Enter S3 path (e.g., s3://bucket/path/to/backup.gz): "
-                read -r s3_path
-                log_prompt "Enter local filename to save as: "
-                read -r local_file
-                download_from_s3 "$s3_path" "$BACKUP_DIR/$local_file"
-                ;;
-            0)
-                log_info "Exiting restore menu"
-                exit 0
-                ;;
-            *)
-                log_error "Invalid choice"
-                ;;
-        esac
-    done
-}
-
-# Main function
-main() {
-    # Check required commands
-    for cmd in psql pg_isready redis-cli tar gzip; do
-        if ! command_exists $cmd; then
-            log_error "Required command not found: $cmd"
-            exit 1
-        fi
-    done
-    
-    # Check if backup directory exists
-    if [ ! -d "$BACKUP_DIR" ]; then
-        log_error "Backup directory not found: $BACKUP_DIR"
-        exit 1
-    fi
-    
-    # Handle command line arguments
-    case "${1:-}" in
-        postgres)
-            if [ -z "${2:-}" ]; then
-                list_backups postgres
-                log_error "Please specify a PostgreSQL backup file"
-                exit 1
-            fi
-            restore_postgres "$2"
-            ;;
-        redis)
-            if [ -z "${2:-}" ]; then
-                list_backups redis
-                log_error "Please specify a Redis backup file"
-                exit 1
-            fi
-            restore_redis "$2"
-            ;;
-        app)
-            if [ -z "${2:-}" ]; then
-                list_backups app
-                log_error "Please specify an application backup file"
-                exit 1
-            fi
-            restore_application "$2"
-            ;;
-        full)
-            full_restore
-            ;;
-        list)
-            list_backups "${2:-all}"
-            ;;
-        *)
-            interactive_restore
-            ;;
-    esac
-}
-
-# Run main function
-main "$@"
diff --git a/apps/backend/scripts/setup-stripe.sh b/apps/backend/scripts/setup-stripe.sh
deleted file mode 100755
index 3e9e673..0000000
--- a/apps/backend/scripts/setup-stripe.sh
+++ /dev/null
@@ -1,265 +0,0 @@
-#!/bin/bash
-
-# Setup Stripe for the UnSearch API
-# This script creates products, prices, and configures webhooks
-
-set -e  # Exit on error
-
-# Colors for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-BLUE='\033[0;34m'
-NC='\033[0m' # No Color
-
-# Logging functions
-log_info() {
-    echo -e "${GREEN}[INFO]${NC} $1"
-}
-
-log_warn() {
-    echo -e "${YELLOW}[WARN]${NC} $1"
-}
-
-log_error() {
-    echo -e "${RED}[ERROR]${NC} $1"
-}
-
-log_prompt() {
-    echo -e "${BLUE}[PROMPT]${NC} $1"
-}
-
-# Check if Stripe CLI is installed
-if ! command -v stripe &> /dev/null; then
-    log_error "Stripe CLI is not installed"
-    log_info "Install it from: https://stripe.com/docs/stripe-cli"
-    exit 1
-fi
-
-# Load environment variables
-if [ -f ".env" ]; then
-    export $(cat .env | grep -v '^#' | xargs)
-fi
-
-# Check if we're in test mode or live mode
-if [ "$1" = "live" ]; then
-    MODE="live"
-    log_warn "Setting up LIVE Stripe products - this will charge real money!"
-    log_prompt "Are you sure you want to continue? (yes/no): "
-    read -r response
-    if [ "$response" != "yes" ]; then
-        log_info "Cancelled"
-        exit 0
-    fi
-else
-    MODE="test"
-    log_info "Setting up TEST Stripe products"
-fi
-
-# Login to Stripe
-log_info "Logging into Stripe..."
-stripe login
-
-# Set the API key
-if [ "$MODE" = "live" ]; then
-    stripe config --set live_mode_api_key
-else
-    stripe config --set test_mode_api_key
-fi
-
-# Function to create a product and price
-create_product() {
-    local name=$1
-    local description=$2
-    local price=$3
-    local interval=$4
-    local metadata=$5
-    
-    log_info "Creating product: $name"
-    
-    # Create product
-    PRODUCT_ID=$(stripe products create \
-        --name="$name" \
-        --description="$description" \
-        --metadata="$metadata" \
-        --json | jq -r '.id')
-    
-    log_info "Product created: $PRODUCT_ID"
-    
-    if [ "$price" != "0" ]; then
-        # Create price
-        log_info "Creating price for $name: \$$price/$interval"
-        
-        PRICE_ID=$(stripe prices create \
-            --product="$PRODUCT_ID" \
-            --unit-amount="$((price * 100))" \
-            --currency="usd" \
-            --recurring[interval]="$interval" \
-            --json | jq -r '.id')
-        
-        log_info "Price created: $PRICE_ID"
-    else
-        PRICE_ID="free"
-    fi
-    
-    echo "$PRODUCT_ID:$PRICE_ID"
-}
-
-# Create products and prices
-log_info "========================================="
-log_info "Creating Stripe Products and Prices"
-log_info "========================================="
-
-# Free Plan (no Stripe product needed)
-log_info "Free plan doesn't require Stripe setup"
-
-# Pro Plan - $20/month
-PRO_RESULT=$(create_product \
-    "UnSearch Pro" \
-    "Unlimited searches and scrapes with priority support" \
-    20 \
-    "month" \
-    "plan=pro,search_limit=unlimited,scrape_limit=unlimited")
-
-PRO_PRODUCT_ID=$(echo $PRO_RESULT | cut -d':' -f1)
-PRO_PRICE_ID=$(echo $PRO_RESULT | cut -d':' -f2)
-
-# Enterprise Plan - $100/month (optional)
-ENTERPRISE_RESULT=$(create_product \
-    "UnSearch Enterprise" \
-    "Custom limits, dedicated support, and SLA" \
-    100 \
-    "month" \
-    "plan=enterprise,custom=true")
-
-ENTERPRISE_PRODUCT_ID=$(echo $ENTERPRISE_RESULT | cut -d':' -f1)
-ENTERPRISE_PRICE_ID=$(echo $ENTERPRISE_RESULT | cut -d':' -f2)
-
-# Setup webhook endpoint
-log_info "========================================="
-log_info "Setting up Webhook Endpoint"
-log_info "========================================="
-
-log_prompt "Enter your webhook URL (e.g., https://api.yourdomain.com/api/v1/billing/webhook/stripe): "
-read -r WEBHOOK_URL
-
-if [ ! -z "$WEBHOOK_URL" ]; then
-    WEBHOOK_ENDPOINT=$(stripe webhook_endpoints create \
-        --url="$WEBHOOK_URL" \
-        --enabled-events="customer.subscription.created,customer.subscription.updated,customer.subscription.deleted,invoice.paid,invoice.payment_failed,payment_intent.succeeded,payment_method.attached,checkout.session.completed" \
-        --json | jq -r '.id')
-    
-    WEBHOOK_SECRET=$(stripe webhook_endpoints retrieve "$WEBHOOK_ENDPOINT" --json | jq -r '.secret')
-    
-    log_info "Webhook endpoint created: $WEBHOOK_ENDPOINT"
-    log_info "Webhook secret: $WEBHOOK_SECRET"
-else
-    log_info "Skipping webhook setup"
-    WEBHOOK_SECRET="whsec_test_secret"
-fi
-
-# Setup billing portal
-log_info "========================================="
-log_info "Configuring Customer Portal"
-log_info "========================================="
-
-stripe billing_portal configurations create \
-    --business-profile[headline]="Manage your UnSearch subscription" \
-    --business-profile[privacy-policy-url]="https://yourdomain.com/privacy" \
-    --business-profile[terms-of-service-url]="https://yourdomain.com/terms" \
-    --features[customer-update][enabled]=true \
-    --features[customer-update][allowed-updates]="email,tax_id" \
-    --features[invoice-history][enabled]=true \
-    --features[payment-method-update][enabled]=true \
-    --features[subscription-cancel][enabled]=true \
-    --features[subscription-cancel][mode]="at_period_end" \
-    --features[subscription-pause][enabled]=false \
-    --features[subscription-update][enabled]=true \
-    --features[subscription-update][products]="$PRO_PRODUCT_ID,$ENTERPRISE_PRODUCT_ID" \
-    --default-return-url="https://yourdomain.com/account"
-
-log_info "Customer portal configured"
-
-# Create test customer (optional)
-if [ "$MODE" = "test" ]; then
-    log_info "========================================="
-    log_info "Creating Test Customer"
-    log_info "========================================="
-    
-    TEST_CUSTOMER=$(stripe customers create \
-        --email="test@example.com" \
-        --name="Test User" \
-        --json | jq -r '.id')
-    
-    log_info "Test customer created: $TEST_CUSTOMER"
-    
-    # Attach test payment method
-    TEST_PM=$(stripe payment_methods attach "pm_card_visa" \
-        --customer="$TEST_CUSTOMER" \
-        --json | jq -r '.id')
-    
-    log_info "Test payment method attached"
-    
-    # Create test subscription
-    TEST_SUB=$(stripe subscriptions create \
-        --customer="$TEST_CUSTOMER" \
-        --items[0][price]="$PRO_PRICE_ID" \
-        --payment-behavior="default_incomplete" \
-        --trial-period-days=7 \
-        --json | jq -r '.id')
-    
-    log_info "Test subscription created: $TEST_SUB"
-fi
-
-# Generate .env configuration
-log_info "========================================="
-log_info "Generating Configuration"
-log_info "========================================="
-
-cat > .env.stripe << EOF
-# Stripe Configuration
-# Generated on $(date)
-
-# Mode: $MODE
-
-# API Keys (get from https://dashboard.stripe.com/apikeys)
-STRIPE_SECRET_KEY=sk_${MODE}_...
-STRIPE_PUBLISHABLE_KEY=pk_${MODE}_...
-
-# Webhook Secret
-STRIPE_WEBHOOK_SECRET=$WEBHOOK_SECRET
-
-# Product IDs
-STRIPE_PRODUCT_ID_PRO=$PRO_PRODUCT_ID
-STRIPE_PRODUCT_ID_ENTERPRISE=$ENTERPRISE_PRODUCT_ID
-
-# Price IDs
-STRIPE_PRICE_ID_PRO=$PRO_PRICE_ID
-STRIPE_PRICE_ID_ENTERPRISE=$ENTERPRISE_PRICE_ID
-
-# Test Customer (if created)
-STRIPE_TEST_CUSTOMER_ID=${TEST_CUSTOMER:-}
-EOF
-
-log_info "Configuration saved to .env.stripe"
-
-# Instructions
-log_info "========================================="
-log_info "Setup Complete!"
-log_info "========================================="
-echo
-log_info "Next steps:"
-echo "1. Copy the Stripe keys from your dashboard: https://dashboard.stripe.com/apikeys"
-echo "2. Add the following to your .env file:"
-echo
-cat .env.stripe
-echo
-echo "3. Test the webhook locally using:"
-echo "   stripe listen --forward-to localhost:8000/api/v1/billing/webhook/stripe"
-echo
-echo "4. Create a test subscription:"
-echo "   curl -X POST http://localhost:8000/api/v1/billing/subscription \\"
-echo "     -H 'Authorization: Bearer YOUR_JWT_TOKEN' \\"
-echo "     -d '{\"price_id\": \"$PRO_PRICE_ID\"}'"
-echo
-log_info "Documentation: https://stripe.com/docs"
diff --git a/apps/backend/scripts/setup.sh b/apps/backend/scripts/setup.sh
deleted file mode 100755
index e821b2a..0000000
--- a/apps/backend/scripts/setup.sh
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/bin/bash
-# Setup script for UnSearch API development environment.
-
-set -e
-
-echo "🚀 Setting up UnSearch API development environment..."
-
-# Check if Python 3.11+ is available
-python_version=$(python3 --version 2>&1 | grep -oE '[0-9]+\.[0-9]+' | head -1)
-required_version="3.11"
-
-if [ "$(printf '%s\n' "$required_version" "$python_version" | sort -V | head -n1)" != "$required_version" ]; then
-    echo "❌ Python 3.11+ is required. Found: $python_version"
-    exit 1
-fi
-
-echo "✅ Python version: $python_version"
-
-# Create virtual environment if it doesn't exist
-if [ ! -d "venv" ]; then
-    echo "📦 Creating virtual environment..."
-    python3 -m venv venv
-fi
-
-# Activate virtual environment
-echo "🔌 Activating virtual environment..."
-source venv/bin/activate
-
-# Upgrade pip using the virtual environment's pip
-echo "⬆️ Upgrading pip..."
-./venv/bin/pip install --upgrade pip
-
-# Install dependencies using the virtual environment's pip
-echo "📥 Installing dependencies..."
-./venv/bin/pip install -r requirements.txt
-
-# Copy environment file if it doesn't exist
-if [ ! -f ".env" ]; then
-    echo "📝 Creating .env file from template..."
-    cp .env.example .env
-    echo "⚠️  Please edit .env file with your configuration"
-fi
-
-# Check if Docker is available
-if command -v docker &> /dev/null; then
-    echo "🐳 Docker found"
-    
-    # Check if docker-compose is available
-    if command -v docker-compose &> /dev/null; then
-        echo "🐙 Docker Compose found"
-        echo "💡 You can start services with: docker-compose up -d"
-    fi
-else
-    echo "⚠️  Docker not found. You'll need to set up services manually."
-fi
-
-# Run database migrations
-echo "🗄️ Running database migrations..."
-if command -v alembic &> /dev/null; then
-    if alembic upgrade head; then
-        echo "✅ Database migrations completed successfully"
-    else
-        echo "⚠️  Database migrations failed. This is normal for initial setup without a database."
-        echo "💡 Set up your database and run 'alembic upgrade head' manually when ready."
-    fi
-else
-    echo "⚠️  Alembic not found in PATH. Skipping migrations."
-fi
-
-# Download NLTK data
-echo "📚 Downloading NLTK data..."
-python3 -c "
-import nltk
-try:
-    nltk.download('punkt', quiet=True)
-    nltk.download('stopwords', quiet=True)
-    print('✅ NLTK data downloaded')
-except Exception as e:
-    print(f'⚠️ Could not download NLTK data: {e}')
-"
-
-echo "🎉 Setup complete!"
-echo ""
-echo "Next steps:"
-echo "1. Edit .env file with your configuration"
-echo "2. Start services: docker-compose up -d (if using Docker)"
-echo "3. Run the API: python -m app.main"
-echo "4. Visit http://localhost:8000/docs for API documentation"
-echo ""
-echo "For development:"
-echo "- Run tests: pytest"
-echo "- Start Celery worker: celery -A app.workers.tasks worker --loglevel=info"
-echo "- Monitor with Flower: celery -A app.workers.tasks flower"
diff --git a/apps/backend/scripts/start-all.sh b/apps/backend/scripts/start-all.sh
deleted file mode 100644
index c059c7f..0000000
--- a/apps/backend/scripts/start-all.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/bin/bash
-# Start SearXNG, initialize services, and run API/workers locally
-
-set -euo pipefail
-
-ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd)
-cd "$ROOT_DIR"
-
-# Load env if present
-if [ -f .env ]; then
-	set -o allexport
-	source .env
-	set +o allexport
-fi
-
-# Sensible defaults
-: "${ENVIRONMENT:=development}"
-: "${SEARXNG_URL:=http://localhost:8080}"
-: "${WORKERS:=4}"
-
-# Ensure CORS JSON lists are valid if provided as plain strings
-export ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-'["*"]'}
-export CORS_METHODS=${CORS_METHODS:-'["GET","POST","PUT","DELETE","OPTIONS"]'}
-export CORS_HEADERS=${CORS_HEADERS:-'["*"]'}
-
-# 1) Start SearXNG via Docker if not already running
-if ! docker ps --format '{{.Names}}' | grep -q '^unsearch-searxng$'; then
-	echo "🐳 Starting SearXNG..."
-	docker compose up -d searxng
-fi
-
-echo "⏳ Waiting for SearXNG to become healthy..."
-SEARX_ATTEMPTS=30
-until curl -fsS "${SEARXNG_URL%/}/healthz" >/dev/null 2>&1 || [ $SEARX_ATTEMPTS -le 0 ]; do
-	SEARX_ATTEMPTS=$((SEARX_ATTEMPTS-1))
-	sleep 2
-	docker ps --format '{{.Names}}: {{.Status}}' | grep unsearch-searxng || true
-	echo -n "."
-
-done
-
-echo ""
-
-# 2) Run DB migrations (handles Neon sslmode gracefully in code)
-echo "🗄️  Running Alembic migrations..."
-poetry run alembic upgrade head || {
-	echo "⚠️ Alembic failed; ensure DATABASE_URL is set and reachable."; exit 1; }
-
-# 3) Start services
-# API
-echo "🚀 Starting API on :8000"
-poetry run uvicorn app.main:app --host 0.0.0.0 --port 8000 &
-API_PID=$!
-
-# Celery worker
-if [ -n "${CELERY_BROKER_URL:-}" ]; then
-	echo "👷 Starting Celery worker"
-	poetry run celery -A app.workers.tasks worker --loglevel=info --concurrency="${WORKERS}" &
-	WORKER_PID=$!
-fi
-
-# Celery beat
-if [ -n "${CELERY_BROKER_URL:-}" ]; then
-	echo "⏰ Starting Celery beat"
-	poetry run celery -A app.workers.tasks beat --loglevel=info &
-	BEAT_PID=$!
-fi
-
-trap 'echo "🧹 Stopping..."; kill ${API_PID:-0} ${WORKER_PID:-0} ${BEAT_PID:-0} 2>/dev/null || true' INT TERM
-
-# 4) Health check loop
-sleep 2
-if curl -fsS http://localhost:8000/health >/dev/null; then
-	echo "✅ API is healthy"
-else
-	echo "❌ API health check failed"
-fi
-
-wait
diff --git a/apps/backend/scripts/test.sh b/apps/backend/scripts/test.sh
deleted file mode 100755
index 9811962..0000000
--- a/apps/backend/scripts/test.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-"""
-Test runner script for UnSearch API.
-"""
-
-set -e
-
-echo "🧪 Running UnSearch API tests..."
-
-# Activate virtual environment if it exists
-if [ -d "venv" ]; then
-    echo "🔌 Activating virtual environment..."
-    source venv/bin/activate
-fi
-
-# Check if pytest is available
-if ! command -v pytest &> /dev/null; then
-    echo "❌ pytest not found. Installing..."
-    pip install pytest pytest-asyncio pytest-cov
-fi
-
-# Set test environment variables
-export TESTING=true
-export DATABASE_URL="sqlite:///test.db"
-export REDIS_URL="redis://localhost:6379/1"  # Use different Redis DB for tests
-export SEARXNG_URL="http://localhost:8080"
-
-echo "🔧 Test configuration:"
-echo "  - Database: $DATABASE_URL"
-echo "  - Redis: $REDIS_URL"
-echo "  - SearXNG: $SEARXNG_URL"
-
-# Run different test suites based on argument
-case "${1:-all}" in
-    "unit")
-        echo "🔬 Running unit tests..."
-        pytest tests/unit/ -v --tb=short
-        ;;
-    "integration")
-        echo "🔗 Running integration tests..."
-        pytest tests/integration/ -v --tb=short
-        ;;
-    "performance")
-        echo "⚡ Running performance tests..."
-        pytest tests/performance/ -v --tb=short -m "not slow"
-        ;;
-    "load")
-        echo "🏋️ Running load tests..."
-        pytest tests/performance/ -v --tb=short -m "slow"
-        ;;
-    "coverage")
-        echo "📊 Running tests with coverage..."
-        pytest tests/ --cov=app --cov-report=html --cov-report=term-missing
-        echo "📈 Coverage report generated in htmlcov/"
-        ;;
-    "quick")
-        echo "⚡ Running quick tests (unit only)..."
-        pytest tests/unit/ -v --tb=line -x
-        ;;
-    "all"|*)
-        echo "🎯 Running all tests..."
-        pytest tests/ -v --tb=short
-        ;;
-esac
-
-# Clean up test artifacts
-echo "🧹 Cleaning up..."
-if [ -f "test.db" ]; then
-    rm test.db
-fi
-
-echo "✅ Tests completed!"
diff --git a/apps/backend/searxng/settings.yml b/apps/backend/searxng/settings.yml
deleted file mode 100644
index 1ccabbd..0000000
--- a/apps/backend/searxng/settings.yml
+++ /dev/null
@@ -1,2841 +0,0 @@
-general:
-  # Debug mode, only for development. Is overwritten by ${SEARXNG_DEBUG}
-  debug: false
-  # displayed name
-  instance_name: "SearXNG"
-  # For example: https://example.com/privacy
-  privacypolicy_url: false
-  # use true to use your own donation page written in searx/info/en/donate.md
-  # use false to disable the donation link
-  donation_url: false
-  # mailto:contact@example.com
-  contact_url: false
-  # record stats
-  enable_metrics: true
-  # expose stats in open metrics format at /metrics
-  # leave empty to disable (no password set)
-  # open_metrics: 
-  open_metrics: ''
-
-brand:
-  new_issue_url: https://github.com/searxng/searxng/issues/new
-  docs_url: https://docs.searxng.org/
-  public_instances: https://searx.space
-  wiki_url: https://github.com/searxng/searxng/wiki
-  issue_url: https://github.com/searxng/searxng/issues
-  # custom:
-  #   maintainer: "Jon Doe"
-  #   # Custom entries in the footer: [title]: [link]
-  #   links:
-  #     Uptime: https://uptime.searxng.org/history/darmarit-org
-  #     About: "https://searxng.org"
-
-search:
-  # Filter results. 0: None, 1: Moderate, 2: Strict
-  safe_search: 0
-  # Existing autocomplete backends: "360search", "baidu", "brave", "dbpedia", "duckduckgo", "google", "yandex",
-  # "mwmbl", "naver", "seznam", "sogou", "startpage", "stract", "swisscows", "quark", "qwant", "wikipedia" -
-  # leave blank to turn it off by default.
-  autocomplete: ""
-  # minimun characters to type before autocompleter starts
-  autocomplete_min: 4
-  # backend for the favicon near URL in search results.
-  # Available resolvers: "allesedv", "duckduckgo", "google", "yandex" - leave blank to turn it off by default.
-  favicon_resolver: ""
-  # Default search language - leave blank to detect from browser information or
-  # use codes from 'languages.py'
-  default_lang: "auto"
-  # max_page: 0  # if engine supports paging, 0 means unlimited numbers of pages
-  # Available languages
-  # languages:
-  #   - all
-  #   - en
-  #   - en-US
-  #   - de
-  #   - it-IT
-  #   - fr
-  #   - fr-BE
-  # ban time in seconds after engine errors
-  ban_time_on_fail: 5
-  # max ban time in seconds after engine errors
-  max_ban_time_on_fail: 120
-  suspended_times:
-    # Engine suspension time after error (in seconds; set to 0 to disable)
-    # For error "Access denied" and "HTTP error [402, 403]"
-    SearxEngineAccessDenied: 86400
-    # For error "CAPTCHA"
-    SearxEngineCaptcha: 86400
-    # For error "Too many request" and "HTTP error 429"
-    SearxEngineTooManyRequests: 3600
-    # Cloudflare CAPTCHA
-    cf_SearxEngineCaptcha: 1296000
-    cf_SearxEngineAccessDenied: 86400
-    # ReCAPTCHA
-    recaptcha_SearxEngineCaptcha: 604800
-
-  # remove format to deny access, use lower case.
-  # formats: [html, csv, json, rss]
-  formats:
-    - html
-
-server:
-  # Is overwritten by ${SEARXNG_PORT} and ${SEARXNG_BIND_ADDRESS}
-  port: 8888
-  bind_address: "127.0.0.1"
-  # public URL of the instance, to ensure correct inbound links. Is overwritten
-  # by ${SEARXNG_BASE_URL}.
-  base_url: false  # "http://example.com/location"
-  # rate limit the number of request on the instance, block some bots.
-  # Is overwritten by ${SEARXNG_LIMITER}
-  limiter: false
-  # enable features designed only for public instances.
-  # Is overwritten by ${SEARXNG_PUBLIC_INSTANCE}
-  public_instance: false
-
-  # If your instance owns a /etc/searxng/settings.yml file, then set the following
-  # values there.
-
-  secret_key: "DJ2CF4Iswmnemfs9FyUACiXbpDn44i8"  # Is overwritten by ${SEARXNG_SECRET}
-  # Proxy image results through SearXNG. Is overwritten by ${SEARXNG_IMAGE_PROXY}
-  image_proxy: false
-  # 1.0 and 1.1 are supported
-  http_protocol_version: "1.0"
-  # POST queries are "more secure!" but are also the source of hard-to-locate
-  # annoyances, which is why GET may be better for end users and their browsers.
-  # see https://github.com/searxng/searxng/pull/3619
-  # Is overwritten by ${SEARXNG_METHOD}
-  method: "POST"
-  default_http_headers:
-    X-Content-Type-Options: nosniff
-    X-Download-Options: noopen
-    X-Robots-Tag: noindex, nofollow
-    Referrer-Policy: no-referrer
-
-valkey:
-  # URL to connect valkey database. Is overwritten by ${SEARXNG_VALKEY_URL}.
-  # https://docs.searxng.org/admin/settings/settings_valkey.html#settings-valkey
-  # url: valkey://localhost:6379/0
-  url: false
-
-ui:
-  # Custom static path - leave it blank if you didn't change
-  static_path: ""
-  # Custom templates path - leave it blank if you didn't change
-  templates_path: ""
-  # query_in_title: When true, the result page's titles contains the query
-  # it decreases the privacy, since the browser can records the page titles.
-  query_in_title: false
-  # infinite_scroll: When true, automatically loads the next page when scrolling to bottom of the current page.
-  infinite_scroll: false
-  # ui theme
-  default_theme: simple
-  # center the results ?
-  center_alignment: false
-  # URL prefix of the internet archive, don't forget trailing slash (if needed).
-  # cache_url: "https://webcache.googleusercontent.com/search?q=cache:"
-  # Default interface locale - leave blank to detect from browser information or
-  # use codes from the 'locales' config section
-  default_locale: ""
-  # Open result links in a new tab by default
-  # results_on_new_tab: false
-  theme_args:
-    # style of simple theme: auto, light, dark
-    simple_style: auto
-  # Perform search immediately if a category selected.
-  # Disable to select multiple categories at once and start the search manually.
-  search_on_category_select: true
-  # Hotkeys: default or vim
-  hotkeys: default
-  # URL formatting: pretty, full or host
-  url_formatting: pretty
-
-# Lock arbitrary settings on the preferences page.
-#
-# preferences:
-#   lock:
-#     - categories
-#     - language
-#     - autocomplete
-#     - favicon
-#     - safesearch
-#     - method
-#     - doi_resolver
-#     - locale
-#     - theme
-#     - results_on_new_tab
-#     - infinite_scroll
-#     - search_on_category_select
-#     - method
-#     - image_proxy
-#     - query_in_title
-
-# communication with search engines
-#
-outgoing:
-  # default timeout in seconds, can be override by engine
-  request_timeout: 3.0
-  # the maximum timeout in seconds
-  # max_request_timeout: 10.0
-  # suffix of searxng_useragent, could contain information like an email address
-  # to the administrator
-  useragent_suffix: ""
-  # The maximum number of concurrent connections that may be established.
-  pool_connections: 100
-  # Allow the connection pool to maintain keep-alive connections below this
-  # point.
-  pool_maxsize: 20
-  # See https://www.python-httpx.org/http2/
-  enable_http2: true
-  # uncomment below section if you want to use a custom server certificate
-  # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults
-  # and https://www.python-httpx.org/compatibility/#ssl-configuration
-  #  verify: ~/.mitmproxy/mitmproxy-ca-cert.cer
-  #
-  # uncomment below section if you want to use a proxyq see: SOCKS proxies
-  #   https://2.python-requests.org/en/latest/user/advanced/#proxies
-  # are also supported: see
-  #   https://2.python-requests.org/en/latest/user/advanced/#socks
-  #
-  #  proxies:
-  #    all://:
-  #      - http://proxy1:8080
-  #      - http://proxy2:8080
-  #
-  #  using_tor_proxy: true
-  #
-  # Extra seconds to add in order to account for the time taken by the proxy
-  #
-  #  extra_proxy_timeout: 10
-  #
-  # uncomment below section only if you have more than one network interface
-  # which can be the source of outgoing search requests
-  #
-  #  source_ips:
-  #    - 1.1.1.1
-  #    - 1.1.1.2
-  #    - fe80::/126
-
-# Plugin configuration, for more details see
-#   https://docs.searxng.org/admin/settings/settings_plugins.html
-#
-plugins:
-
-  searx.plugins.calculator.SXNGPlugin:
-    active: true
-
-  searx.plugins.hash_plugin.SXNGPlugin:
-    active: true
-
-  searx.plugins.self_info.SXNGPlugin:
-    active: true
-
-  searx.plugins.unit_converter.SXNGPlugin:
-    active: true
-
-  searx.plugins.ahmia_filter.SXNGPlugin:
-    active: true
-
-  searx.plugins.hostnames.SXNGPlugin:
-    active: true
-
-  searx.plugins.time_zone.SXNGPlugin:
-    active: true
-
-  searx.plugins.oa_doi_rewrite.SXNGPlugin:
-    active: false
-
-  searx.plugins.tor_check.SXNGPlugin:
-    active: false
-
-  searx.plugins.tracker_url_remover.SXNGPlugin:
-    active: true
-
-
-# Configuration of the "Hostnames plugin":
-#
-# hostnames:
-#   replace:
-#     '(.*\.)?youtube\.com$': 'yt.example.com'
-#     '(.*\.)?youtu\.be$': 'yt.example.com'
-#     '(.*\.)?reddit\.com$': 'teddit.example.com'
-#     '(.*\.)?redd\.it$': 'teddit.example.com'
-#     '(www\.)?twitter\.com$': 'nitter.example.com'
-#   remove:
-#     - '(.*\.)?facebook.com$'
-#   low_priority:
-#     - '(.*\.)?google(\..*)?$'
-#   high_priority:
-#     - '(.*\.)?wikipedia.org$'
-#
-# Alternatively you can use external files for configuring the "Hostnames plugin":
-#
-# hostnames:
-#  replace: 'rewrite-hosts.yml'
-#
-# Content of 'rewrite-hosts.yml' (place the file in the same directory as 'settings.yml'):
-# '(.*\.)?youtube\.com$': 'yt.example.com'
-# '(.*\.)?youtu\.be$': 'yt.example.com'
-#
-
-checker:
-  # disable checker when in debug mode
-  off_when_debug: true
-
-  # use "scheduling: {}" to disable scheduling
-  # scheduling: interval or int
-
-  # to activate the scheduler:
-  # * uncomment "scheduling" section
-  # * add "cache2 = name=searxngcache,items=2000,blocks=2000,blocksize=4096,bitmap=1"
-  #   to your uwsgi.ini
-
-  # scheduling:
-  #   start_after: [300, 1800]  # delay to start the first run of the checker
-  #   every: [86400, 90000]     # how often the checker runs
-
-  # additional tests: only for the YAML anchors (see the engines section)
-  #
-  additional_tests:
-    rosebud: &test_rosebud
-      matrix:
-        query: rosebud
-        lang: en
-      result_container:
-        - not_empty
-        - ['one_title_contains', 'citizen kane']
-      test:
-        - unique_results
-
-    android: &test_android
-      matrix:
-        query: ['android']
-        lang: ['en', 'de', 'fr', 'zh-CN']
-      result_container:
-        - not_empty
-        - ['one_title_contains', 'google']
-      test:
-        - unique_results
-
-  # tests: only for the YAML anchors (see the engines section)
-  tests:
-    infobox: &tests_infobox
-      infobox:
-        matrix:
-          query: ["linux", "new york", "bbc"]
-        result_container:
-          - has_infobox
-
-categories_as_tabs:
-  general:
-  images:
-  videos:
-  news:
-  map:
-  music:
-  it:
-  science:
-  files:
-  social media:
-
-engines:
-  - name: 360search
-    engine: 360search
-    shortcut: 360so
-    disabled: true
-
-  - name: 360search videos
-    engine: 360search_videos
-    shortcut: 360sov
-    disabled: true
-
-  - name: 9gag
-    engine: 9gag
-    shortcut: 9g
-    disabled: true
-
-  - name: acfun
-    engine: acfun
-    shortcut: acf
-    disabled: true
-
-  - name: adobe stock
-    engine: adobe_stock
-    shortcut: asi
-    categories: ["images"]
-    # https://docs.searxng.org/dev/engines/online/adobe_stock.html
-    adobe_order: relevance
-    adobe_content_types: ["photo", "illustration", "zip_vector", "template", "3d", "image"]
-    timeout: 6
-    disabled: true
-
-  - name: adobe stock video
-    engine: adobe_stock
-    shortcut: asv
-    network: adobe stock
-    categories: ["videos"]
-    adobe_order: relevance
-    adobe_content_types: ["video"]
-    timeout: 6
-    disabled: true
-
-  - name: adobe stock audio
-    engine: adobe_stock
-    shortcut: asa
-    network: adobe stock
-    categories: ["music"]
-    adobe_order: relevance
-    adobe_content_types: ["audio"]
-    timeout: 6
-    disabled: true
-
-  - name: alexandria
-    engine: json_engine
-    shortcut: alx
-    categories: general
-    paging: true
-    search_url: https://api.alexandria.org/?a=1&q={query}&p={pageno}
-    results_query: results
-    title_query: title
-    url_query: url
-    content_query: snippet
-    timeout: 1.5
-    disabled: true
-    about:
-      website: https://alexandria.org/
-      official_api_documentation: https://github.com/alexandria-org/alexandria-api/raw/master/README.md
-      use_official_api: true
-      require_api_key: false
-      results: JSON
-
-  # - name: astrophysics data system
-  #   engine: astrophysics_data_system
-  #   sort: asc
-  #   weight: 5
-  #   categories: [science]
-  #   api_key: your-new-key
-  #   shortcut: ads
-
-  - name: alpine linux packages
-    engine: alpinelinux
-    disabled: true
-    shortcut: alp
-
-  - name: annas archive
-    engine: annas_archive
-    disabled: true
-    shortcut: aa
-
-  - name: ansa
-    engine: ansa
-    shortcut: ans
-    disabled: true
-
-  # - name: annas articles
-  #   engine: annas_archive
-  #   shortcut: aaa
-  #   # https://docs.searxng.org/dev/engines/online/annas_archive.html
-  #   aa_content: 'magazine' # book_fiction, book_unknown, book_nonfiction, book_comic
-  #   aa_ext: 'pdf'  # pdf, epub, ..
-  #   aa_sort: oldest'  # newest, oldest, largest, smallest
-
-  - name: apk mirror
-    engine: apkmirror
-    timeout: 4.0
-    shortcut: apkm
-    disabled: true
-
-  - name: apple app store
-    engine: apple_app_store
-    shortcut: aps
-    disabled: true
-
-  # Requires Tor
-  - name: ahmia
-    engine: ahmia
-    categories: onions
-    enable_http: true
-    shortcut: ah
-
-  - name: anaconda
-    engine: xpath
-    paging: true
-    first_page_num: 0
-    search_url: https://anaconda.org/search?q={query}&page={pageno}
-    results_xpath: //tbody/tr
-    url_xpath: ./td/h5/a[last()]/@href
-    title_xpath: ./td/h5
-    content_xpath: ./td[h5]/text()
-    categories: it
-    timeout: 6.0
-    shortcut: conda
-    disabled: true
-
-  - name: arch linux wiki
-    engine: archlinux
-    shortcut: al
-
-  - name: nixos wiki
-    engine: mediawiki
-    shortcut: nixw
-    base_url: https://wiki.nixos.org/
-    search_type: text
-    disabled: true
-    categories: [it, software wikis]
-
-  - name: artic
-    engine: artic
-    shortcut: arc
-    timeout: 4.0
-
-  - name: arxiv
-    engine: arxiv
-    shortcut: arx
-    timeout: 4.0
-
-  - name: ask
-    engine: ask
-    shortcut: ask
-    disabled: true
-
-  # tmp suspended:  dh key too small
-  # - name: base
-  #   engine: base
-  #   shortcut: bs
-
-  - name: bandcamp
-    engine: bandcamp
-    shortcut: bc
-    categories: music
-
-  - name: baidu
-    baidu_category: general
-    categories: [general]
-    engine: baidu
-    shortcut: bd
-    disabled: true
-
-  - name: baidu images
-    baidu_category: images
-    categories: [images]
-    engine: baidu
-    shortcut: bdi
-    disabled: true
-
-  - name: baidu kaifa
-    baidu_category: it
-    categories: [it]
-    engine: baidu
-    shortcut: bdk
-    disabled: true
-
-  - name: wikipedia
-    engine: wikipedia
-    shortcut: wp
-    # add "list" to the array to get results in the results list
-    display_type: ["infobox"]
-    categories: [general]
-
-  - name: bilibili
-    engine: bilibili
-    shortcut: bil
-    disabled: true
-
-  - name: bing
-    engine: bing
-    shortcut: bi
-    disabled: true
-
-  - name: bing images
-    engine: bing_images
-    shortcut: bii
-
-  - name: bing news
-    engine: bing_news
-    shortcut: bin
-
-  - name: bing videos
-    engine: bing_videos
-    shortcut: biv
-
-  - name: bitchute
-    engine: bitchute
-    shortcut: bit
-    disabled: true
-
-  - name: bitbucket
-    engine: xpath
-    paging: true
-    search_url: https://bitbucket.org/repo/all/{pageno}?name={query}
-    url_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]/@href
-    title_xpath: //article[@class="repo-summary"]//a[@class="repo-link"]
-    content_xpath: //article[@class="repo-summary"]/p
-    categories: [it, repos]
-    timeout: 4.0
-    disabled: true
-    shortcut: bb
-    about:
-      website: https://bitbucket.org/
-      wikidata_id: Q2493781
-      official_api_documentation: https://developer.atlassian.com/bitbucket
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: bpb
-    engine: bpb
-    shortcut: bpb
-    disabled: true
-
-  - name: btdigg
-    engine: btdigg
-    shortcut: bt
-    disabled: true
-
-  - name: openverse
-    engine: openverse
-    categories: images
-    shortcut: opv
-
-  - name: media.ccc.de
-    engine: ccc_media
-    shortcut: c3tv
-    # We don't set language: de here because media.ccc.de is not just
-    # for a German audience. It contains many English videos and many
-    # German videos have English subtitles.
-    disabled: true
-
-  - name: chefkoch
-    engine: chefkoch
-    shortcut: chef
-    # to show premium or plus results too:
-    # skip_premium: false
-
-  # WARNING: links from chinaso.com voilate users privacy
-  # Before activate these engines its mandatory to read
-  # - https://github.com/searxng/searxng/issues/4694
-  # - https://docs.searxng.org/dev/engines/online/chinaso.html
-
-  - name: chinaso news
-    engine: chinaso
-    shortcut: chinaso
-    categories: [news]
-    chinaso_category: news
-    chinaso_news_source: all
-    disabled: true
-    inactive: true
-
-  - name: chinaso images
-    engine: chinaso
-    network: chinaso news
-    shortcut: chinasoi
-    categories: [images]
-    chinaso_category: images
-    disabled: true
-    inactive: true
-
-  - name: chinaso videos
-    engine: chinaso
-    network: chinaso news
-    shortcut: chinasov
-    categories: [videos]
-    chinaso_category: videos
-    disabled: true
-    inactive: true
-
-  - name: cloudflareai
-    engine: cloudflareai
-    shortcut: cfai
-    # get api token and accont id from https://developers.cloudflare.com/workers-ai/get-started/rest-api/
-    cf_account_id: 'your_cf_accout_id'
-    cf_ai_api: 'your_cf_api'
-    # create your ai gateway by https://developers.cloudflare.com/ai-gateway/get-started/creating-gateway/
-    cf_ai_gateway: 'your_cf_ai_gateway_name'
-    # find the model name from https://developers.cloudflare.com/workers-ai/models/#text-generation
-    cf_ai_model: 'ai_model_name'
-    # custom your preferences
-    # cf_ai_model_display_name: 'Cloudflare AI'
-    # cf_ai_model_assistant: 'prompts_for_assistant_role'
-    # cf_ai_model_system: 'prompts_for_system_role'
-    timeout: 30
-    disabled: true
-
-  # - name: core.ac.uk
-  #   engine: core
-  #   categories: science
-  #   shortcut: cor
-  #   # get your API key from: https://core.ac.uk/api-keys/register/
-  #   api_key: 'unset'
-
-  - name: cppreference
-    engine: cppreference
-    shortcut: cpp
-    paging: false
-    disabled: true
-
-  - name: crossref
-    engine: crossref
-    shortcut: cr
-    timeout: 30
-    disabled: true
-
-  - name: crowdview
-    engine: json_engine
-    shortcut: cv
-    categories: general
-    paging: false
-    search_url: https://crowdview-next-js.onrender.com/api/search-v3?query={query}
-    results_query: results
-    url_query: link
-    title_query: title
-    content_query: snippet
-    title_html_to_text: true
-    content_html_to_text: true
-    disabled: true
-    about:
-      website: https://crowdview.ai/
-
-  - name: yep
-    engine: yep
-    shortcut: yep
-    categories: general
-    search_type: web
-    timeout: 5
-    disabled: true
-
-  - name: yep images
-    engine: yep
-    shortcut: yepi
-    categories: images
-    search_type: images
-    disabled: true
-
-  - name: yep news
-    engine: yep
-    shortcut: yepn
-    categories: news
-    search_type: news
-    disabled: true
-
-  - name: currency
-    engine: currency_convert
-    categories: general
-    shortcut: cc
-
-  - name: deezer
-    engine: deezer
-    shortcut: dz
-    disabled: true
-
-  - name: destatis
-    engine: destatis
-    shortcut: destat
-    disabled: true
-
-  - name: deviantart
-    engine: deviantart
-    shortcut: da
-    timeout: 3.0
-
-  - name: ddg definitions
-    engine: duckduckgo_definitions
-    shortcut: ddd
-    weight: 2
-    disabled: true
-    tests: *tests_infobox
-
-  # cloudflare protected
-  # - name: digbt
-  #   engine: digbt
-  #   shortcut: dbt
-  #   timeout: 6.0
-  #   disabled: true
-
-  - name: docker hub
-    engine: docker_hub
-    shortcut: dh
-    categories: [it, packages]
-
-  - name: encyclosearch
-    engine: json_engine
-    shortcut: es
-    categories: general
-    paging: true
-    search_url: https://encyclosearch.org/encyclosphere/search?q={query}&page={pageno}&resultsPerPage=15
-    results_query: Results
-    url_query: SourceURL
-    title_query: Title
-    content_query: Description
-    disabled: true
-    about:
-      website: https://encyclosearch.org
-      official_api_documentation: https://encyclosearch.org/docs/#/rest-api
-      use_official_api: true
-      require_api_key: false
-      results: JSON
-
-  - name: erowid
-    engine: xpath
-    paging: true
-    first_page_num: 0
-    page_size: 30
-    search_url: https://www.erowid.org/search.php?q={query}&s={pageno}
-    url_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/@href
-    title_xpath: //dl[@class="results-list"]/dt[@class="result-title"]/a/text()
-    content_xpath: //dl[@class="results-list"]/dd[@class="result-details"]
-    categories: []
-    shortcut: ew
-    disabled: true
-    about:
-      website: https://www.erowid.org/
-      wikidata_id: Q1430691
-      official_api_documentation:
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  # - name: elasticsearch
-  #   shortcut: els
-  #   engine: elasticsearch
-  #   base_url: http://localhost:9200
-  #   username: elastic
-  #   password: changeme
-  #   index: my-index
-  #   enable_http: true
-  #   # available options: match, simple_query_string, term, terms, custom
-  #   query_type: match
-  #   # if query_type is set to custom, provide your query here
-  #   # custom_query_json: {"query":{"match_all": {}}}
-  #   # show_metadata: false
-  #   disabled: true
-
-  - name: wikidata
-    engine: wikidata
-    shortcut: wd
-    timeout: 3.0
-    weight: 2
-    # add "list" to the array to get results in the results list
-    display_type: ["infobox"]
-    tests: *tests_infobox
-    categories: [general]
-
-  - name: duckduckgo
-    engine: duckduckgo
-    shortcut: ddg
-
-  - name: duckduckgo images
-    engine: duckduckgo_extra
-    categories: [images, web]
-    ddg_category: images
-    shortcut: ddi
-    disabled: true
-
-  - name: duckduckgo videos
-    engine: duckduckgo_extra
-    categories: [videos, web]
-    ddg_category: videos
-    shortcut: ddv
-    disabled: true
-
-  - name: duckduckgo news
-    engine: duckduckgo_extra
-    categories: [news, web]
-    ddg_category: news
-    shortcut: ddn
-    disabled: true
-
-  - name: duckduckgo weather
-    engine: duckduckgo_weather
-    shortcut: ddw
-    disabled: true
-
-  - name: apple maps
-    engine: apple_maps
-    shortcut: apm
-    disabled: true
-    timeout: 5.0
-
-  - name: emojipedia
-    engine: emojipedia
-    timeout: 4.0
-    shortcut: em
-    disabled: true
-
-  - name: tineye
-    engine: tineye
-    shortcut: tin
-    timeout: 9.0
-    disabled: true
-
-  - name: etymonline
-    engine: xpath
-    paging: true
-    search_url: https://etymonline.com/search?page={pageno}&q={query}
-    url_xpath: //a[contains(@class, "word__name--")]/@href
-    title_xpath: //a[contains(@class, "word__name--")]
-    content_xpath: //section[contains(@class, "word__defination")]
-    first_page_num: 1
-    shortcut: et
-    categories: [dictionaries]
-    about:
-      website: https://www.etymonline.com/
-      wikidata_id: Q1188617
-      official_api_documentation:
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  # - name: ebay
-  #   engine: ebay
-  #   shortcut: eb
-  #   base_url: 'https://www.ebay.com'
-  #   disabled: true
-  #   timeout: 5
-
-  - name: 1x
-    engine: www1x
-    shortcut: 1x
-    timeout: 3.0
-    disabled: true
-
-  - name: fdroid
-    engine: fdroid
-    shortcut: fd
-    disabled: true
-
-  - name: findthatmeme
-    engine: findthatmeme
-    shortcut: ftm
-    disabled: true
-
-  - name: flickr
-    categories: images
-    shortcut: fl
-    # You can use the engine using the official stable API, but you need an API
-    # key, see: https://www.flickr.com/services/apps/create/
-    # engine: flickr
-    # api_key: 'apikey' # required!
-    # Or you can use the html non-stable engine, activated by default
-    engine: flickr_noapi
-
-  - name: free software directory
-    engine: mediawiki
-    shortcut: fsd
-    categories: [it, software wikis]
-    base_url: https://directory.fsf.org/
-    search_type: title
-    timeout: 5.0
-    disabled: true
-    about:
-      website: https://directory.fsf.org/
-      wikidata_id: Q2470288
-
-  # - name: freesound
-  #   engine: freesound
-  #   shortcut: fnd
-  #   disabled: true
-  #   timeout: 15.0
-  # API key required, see: https://freesound.org/docs/api/overview.html
-  #   api_key: MyAPIkey
-
-  - name: frinkiac
-    engine: frinkiac
-    shortcut: frk
-    disabled: true
-
-  - name: fyyd
-    engine: fyyd
-    shortcut: fy
-    timeout: 8.0
-    disabled: true
-
-  - name: geizhals
-    engine: geizhals
-    shortcut: geiz
-    disabled: true
-
-  - name: genius
-    engine: genius
-    shortcut: gen
-
-  - name: gentoo
-    engine: mediawiki
-    shortcut: ge
-    categories: ["it", "software wikis"]
-    base_url: "https://wiki.gentoo.org/"
-    api_path: "api.php"
-    search_type: text
-    timeout: 10
-
-  - name: gitlab
-    engine: gitlab
-    base_url: https://gitlab.com
-    shortcut: gl
-    disabled: true
-    about:
-      website: https://gitlab.com/
-      wikidata_id: Q16639197
-
-  # - name: gnome
-  #   engine: gitlab
-  #   base_url: https://gitlab.gnome.org
-  #   shortcut: gn
-  #   about:
-  #     website: https://gitlab.gnome.org
-  #     wikidata_id: Q44316
-
-  - name: github
-    engine: github
-    shortcut: gh
-
-  - name: github code
-    engine: github_code
-    shortcut: ghc
-    disabled: true
-    ghc_auth:
-      # type is one of:
-      # * none
-      # * personal_access_token
-      # * bearer
-      # When none is passed, the token is not requried.
-      type: "none"
-      token: "token"
-    # specify whether to highlight the matching lines to the query
-    ghc_highlight_matching_lines: true
-    ghc_strip_new_lines: true
-    ghc_strip_whitespace: false
-    timeout: 10.0
-
-  - name: codeberg
-    # https://docs.searxng.org/dev/engines/online/gitea.html
-    engine: gitea
-    base_url: https://codeberg.org
-    shortcut: cb
-    disabled: true
-
-  - name: gitea.com
-    engine: gitea
-    base_url: https://gitea.com
-    shortcut: gitea
-    disabled: true
-
-  - name: goodreads
-    engine: goodreads
-    shortcut: good
-    timeout: 4.0
-    disabled: true
-
-  - name: google
-    engine: google
-    shortcut: go
-    # additional_tests:
-    #   android: *test_android
-
-  - name: google images
-    engine: google_images
-    shortcut: goi
-    # additional_tests:
-    #   android: *test_android
-    #   dali:
-    #     matrix:
-    #       query: ['Dali Christ']
-    #       lang: ['en', 'de', 'fr', 'zh-CN']
-    #     result_container:
-    #       - ['one_title_contains', 'Salvador']
-
-  - name: google news
-    engine: google_news
-    shortcut: gon
-    # additional_tests:
-    #   android: *test_android
-
-  - name: google videos
-    engine: google_videos
-    shortcut: gov
-    # additional_tests:
-    #   android: *test_android
-
-  - name: google scholar
-    engine: google_scholar
-    shortcut: gos
-
-  - name: google play apps
-    engine: google_play
-    categories: [files, apps]
-    shortcut: gpa
-    play_categ: apps
-    disabled: true
-
-  - name: google play movies
-    engine: google_play
-    categories: videos
-    shortcut: gpm
-    play_categ: movies
-    disabled: true
-
-  - name: material icons
-    engine: material_icons
-    shortcut: mi
-    disabled: true
-
-  - name: habrahabr
-    engine: xpath
-    paging: true
-    search_url: https://habr.com/en/search/page{pageno}/?q={query}
-    results_xpath: //article[contains(@class, "tm-articles-list__item")]
-    url_xpath: .//a[@class="tm-title__link"]/@href
-    title_xpath: .//a[@class="tm-title__link"]
-    content_xpath: .//div[contains(@class, "article-formatted-body")]
-    categories: it
-    timeout: 4.0
-    disabled: true
-    shortcut: habr
-    about:
-      website: https://habr.com/
-      wikidata_id: Q4494434
-      official_api_documentation: https://habr.com/en/docs/help/api/
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: hackernews
-    engine: hackernews
-    shortcut: hn
-    disabled: true
-
-  - name: hex
-    engine: hex
-    shortcut: hex
-    disabled: true
-    # Valid values: name inserted_at updated_at total_downloads recent_downloads
-    sort_criteria: "recent_downloads"
-    page_size: 10
-
-  - name: crates.io
-    engine: crates
-    shortcut: crates
-    disabled: true
-    timeout: 6.0
-
-  - name: hoogle
-    engine: xpath
-    search_url: https://hoogle.haskell.org/?hoogle={query}
-    results_xpath: '//div[@class="result"]'
-    title_xpath: './/div[@class="ans"]//a'
-    url_xpath: './/div[@class="ans"]//a/@href'
-    content_xpath: './/div[@class="from"]'
-    page_size: 20
-    categories: [it, packages]
-    shortcut: ho
-    about:
-      website: https://hoogle.haskell.org/
-      wikidata_id: Q34010
-      official_api_documentation: https://hackage.haskell.org/api
-      use_official_api: false
-      require_api_key: false
-      results: JSON
-
-  - name: il post
-    engine: il_post
-    shortcut: pst
-    disabled: true
-
-  - name: huggingface
-    engine: huggingface
-    shortcut: hf
-    disabled: true
-
-  - name: huggingface datasets
-    huggingface_endpoint: datasets
-    engine: huggingface
-    shortcut: hfd
-    disabled: true
-
-  - name: huggingface spaces
-    huggingface_endpoint: spaces
-    engine: huggingface
-    shortcut: hfs
-    disabled: true
-
-  - name: imdb
-    engine: imdb
-    shortcut: imdb
-    timeout: 6.0
-    disabled: true
-
-  - name: imgur
-    engine: imgur
-    shortcut: img
-    disabled: true
-
-  - name: ina
-    engine: ina
-    shortcut: in
-    timeout: 6.0
-    disabled: true
-
-  # - name: invidious
-  #   engine: invidious
-  #   # if you want to use invidious with SearXNG you should setup one locally
-  #   # https://github.com/searxng/searxng/issues/2722#issuecomment-2884993248
-  #   base_url:
-  #     - https://invidious.example1.com
-  #     - https://invidious.example2.com
-  #   shortcut: iv
-  #   timeout: 3.0
-
-  - name: ipernity
-    engine: ipernity
-    shortcut: ip
-    disabled: true
-
-  - name: iqiyi
-    engine: iqiyi
-    shortcut: iq
-    disabled: true
-
-  - name: jisho
-    engine: jisho
-    shortcut: js
-    timeout: 3.0
-    disabled: true
-
-  - name: kickass
-    engine: kickass
-    base_url:
-      - https://kickasstorrents.to
-      - https://kickasstorrents.cr
-      - https://kickasstorrent.cr
-      - https://kickass.sx
-      - https://kat.am
-    shortcut: kc
-    timeout: 4.0
-
-  - name: lemmy communities
-    engine: lemmy
-    lemmy_type: Communities
-    shortcut: leco
-
-  - name: lemmy users
-    engine: lemmy
-    network: lemmy communities
-    lemmy_type: Users
-    shortcut: leus
-
-  - name: lemmy posts
-    engine: lemmy
-    network: lemmy communities
-    lemmy_type: Posts
-    shortcut: lepo
-
-  - name: lemmy comments
-    engine: lemmy
-    network: lemmy communities
-    lemmy_type: Comments
-    shortcut: lecom
-
-  - name: library genesis
-    engine: xpath
-    # search_url: https://libgen.is/search.php?req={query}
-    search_url: https://libgen.rs/search.php?req={query}
-    url_xpath: //a[contains(@href,"book/index.php?md5")]/@href
-    title_xpath: //a[contains(@href,"book/")]/text()[1]
-    content_xpath: //td/a[1][contains(@href,"=author")]/text()
-    categories: files
-    timeout: 7.0
-    disabled: true
-    shortcut: lg
-    about:
-      website: https://libgen.fun/
-      wikidata_id: Q22017206
-      official_api_documentation:
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: z-library
-    engine: zlibrary
-    shortcut: zlib
-    categories: files
-    timeout: 7.0
-    disabled: true
-
-  - name: library of congress
-    engine: loc
-    shortcut: loc
-    categories: images
-    disabled: true
-
-  - name: libretranslate
-    engine: libretranslate
-    # https://github.com/LibreTranslate/LibreTranslate?tab=readme-ov-file#mirrors
-    base_url:
-      - https://libretranslate.com/translate
-    # api_key: abc123
-    shortcut: lt
-    disabled: true
-
-  - name: lingva
-    engine: lingva
-    shortcut: lv
-    # set lingva instance in url, by default it will use the official instance
-    # url: https://lingva.thedaviddelta.com
-
-  - name: lobste.rs
-    engine: xpath
-    search_url: https://lobste.rs/search?q={query}&what=stories&order=relevance
-    results_xpath: //li[contains(@class, "story")]
-    url_xpath: .//a[@class="u-url"]/@href
-    title_xpath: .//a[@class="u-url"]
-    content_xpath: .//a[@class="domain"]
-    categories: it
-    shortcut: lo
-    timeout: 5.0
-    disabled: true
-    about:
-      website: https://lobste.rs/
-      wikidata_id: Q60762874
-      official_api_documentation:
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: marginalia
-    engine: marginalia
-    shortcut: mar
-    # To get an API key, please follow the instructions at
-    # - https://about.marginalia-search.com/article/api/
-    # api_key: ...
-    disabled: true
-    inactive: true
-
-  - name: mastodon users
-    engine: mastodon
-    mastodon_type: accounts
-    base_url: https://mastodon.social
-    shortcut: mau
-
-  - name: mastodon hashtags
-    engine: mastodon
-    mastodon_type: hashtags
-    base_url: https://mastodon.social
-    shortcut: mah
-
-  # - name: matrixrooms
-  #   engine: mrs
-  #   # https://docs.searxng.org/dev/engines/online/mrs.html
-  #   # base_url: https://mrs-api-host
-  #   shortcut: mtrx
-  #   disabled: true
-
-  - name: mdn
-    shortcut: mdn
-    engine: json_engine
-    categories: [it]
-    paging: true
-    search_url: https://developer.mozilla.org/api/v1/search?q={query}&page={pageno}
-    results_query: documents
-    url_query: mdn_url
-    url_prefix: https://developer.mozilla.org
-    title_query: title
-    content_query: summary
-    about:
-      website: https://developer.mozilla.org
-      wikidata_id: Q3273508
-      official_api_documentation: null
-      use_official_api: false
-      require_api_key: false
-      results: JSON
-
-  - name: metacpan
-    engine: metacpan
-    shortcut: cpan
-    disabled: true
-    number_of_results: 20
-
-  # https://docs.searxng.org/dev/engines/offline/search-indexer-engines.html#module-searx.engines.meilisearch
-  # - name: meilisearch
-  #   engine: meilisearch
-  #   shortcut: mes
-  #   enable_http: true
-  #   base_url: http://localhost:7700
-  #   index: my-index
-  #   auth_key: Bearer XXXX
-
-  - name: microsoft learn
-    engine: microsoft_learn
-    shortcut: msl
-    disabled: true
-
-  - name: mixcloud
-    engine: mixcloud
-    shortcut: mc
-
-  # MongoDB engine
-  # Required dependency: pymongo
-  # - name: mymongo
-  #   engine: mongodb
-  #   shortcut: md
-  #   exact_match_only: false
-  #   host: '127.0.0.1'
-  #   port: 27017
-  #   enable_http: true
-  #   results_per_page: 20
-  #   database: 'business'
-  #   collection: 'reviews'  # name of the db collection
-  #   key: 'name'  # key in the collection to search for
-
-  - name: mozhi
-    engine: mozhi
-    base_url:
-      - https://mozhi.aryak.me
-      - https://translate.bus-hit.me
-      - https://nyc1.mz.ggtyler.dev
-    # mozhi_engine: google - see https://mozhi.aryak.me for supported engines
-    timeout: 4.0
-    shortcut: mz
-    disabled: true
-
-  - name: mwmbl
-    engine: mwmbl
-    # api_url: https://api.mwmbl.org
-    shortcut: mwm
-    disabled: true
-
-  - name: niconico
-    engine: niconico
-    shortcut: nico
-    disabled: true
-
-  - name: npm
-    engine: npm
-    shortcut: npm
-    timeout: 5.0
-    disabled: true
-
-  - name: nyaa
-    engine: nyaa
-    shortcut: nt
-    disabled: true
-
-  - name: mankier
-    engine: json_engine
-    search_url: https://www.mankier.com/api/v2/mans/?q={query}
-    results_query: results
-    url_query: url
-    title_query: name
-    content_query: description
-    categories: it
-    shortcut: man
-    about:
-      website: https://www.mankier.com/
-      official_api_documentation: https://www.mankier.com/api
-      use_official_api: true
-      require_api_key: false
-      results: JSON
-
-  # https://docs.searxng.org/dev/engines/online/mullvad_leta.html
-  - name: mullvadleta
-    engine: mullvad_leta
-    disabled: true
-    leta_engine: google
-    categories: [general, web]
-    shortcut: ml
-
-  - name: mullvadleta brave
-    engine: mullvad_leta
-    network: mullvadleta
-    disabled: true
-    leta_engine: brave
-    categories: [general, web]
-    shortcut: mlb
-
-  - name: odysee
-    engine: odysee
-    shortcut: od
-    disabled: true
-
-  - name: ollama
-    engine: ollama
-    shortcut: ollama
-    disabled: true
-
-  - name: openairedatasets
-    engine: json_engine
-    paging: true
-    search_url: https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query}
-    results_query: response/results/result
-    url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$
-    title_query: metadata/oaf:entity/oaf:result/title/$
-    content_query: metadata/oaf:entity/oaf:result/description/$
-    content_html_to_text: true
-    categories: "science"
-    shortcut: oad
-    timeout: 5.0
-    about:
-      website: https://www.openaire.eu/
-      wikidata_id: Q25106053
-      official_api_documentation: https://api.openaire.eu/
-      use_official_api: false
-      require_api_key: false
-      results: JSON
-
-  - name: openairepublications
-    engine: json_engine
-    paging: true
-    search_url: https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query}
-    results_query: response/results/result
-    url_query: metadata/oaf:entity/oaf:result/children/instance/webresource/url/$
-    title_query: metadata/oaf:entity/oaf:result/title/$
-    content_query: metadata/oaf:entity/oaf:result/description/$
-    content_html_to_text: true
-    categories: science
-    shortcut: oap
-    timeout: 5.0
-    about:
-      website: https://www.openaire.eu/
-      wikidata_id: Q25106053
-      official_api_documentation: https://api.openaire.eu/
-      use_official_api: false
-      require_api_key: false
-      results: JSON
-
-  - name: openalex
-    engine: openalex
-    shortcut: oa
-    # https://docs.searxng.org/dev/engines/online/openalex.html
-    # Recommended by OpenAlex: join the polite pool with an email address
-    # mailto: "[email protected]"
-    timeout: 5.0
-    disabled: true
-
-  - name: openclipart
-    engine: openclipart
-    shortcut: ocl
-    inactive: true
-    disabled: true
-    timeout: 30
-
-  - name: openlibrary
-    engine: openlibrary
-    shortcut: ol
-    timeout: 5
-    disabled: true
-
-  - name: openmeteo
-    engine: open_meteo
-    shortcut: om
-    disabled: true
-
-  # - name: opensemanticsearch
-  #   engine: opensemantic
-  #   shortcut: oss
-  #   base_url: 'http://localhost:8983/solr/opensemanticsearch/'
-
-  - name: openstreetmap
-    engine: openstreetmap
-    shortcut: osm
-
-  - name: openrepos
-    engine: xpath
-    paging: true
-    search_url: https://openrepos.net/search/node/{query}?page={pageno}
-    url_xpath: //li[@class="search-result"]//h3[@class="title"]/a/@href
-    title_xpath: //li[@class="search-result"]//h3[@class="title"]/a
-    content_xpath: //li[@class="search-result"]//div[@class="search-snippet-info"]//p[@class="search-snippet"]
-    categories: files
-    timeout: 4.0
-    disabled: true
-    shortcut: or
-    about:
-      website: https://openrepos.net/
-      wikidata_id:
-      official_api_documentation:
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: packagist
-    engine: json_engine
-    paging: true
-    search_url: https://packagist.org/search.json?q={query}&page={pageno}
-    results_query: results
-    url_query: url
-    title_query: name
-    content_query: description
-    categories: [it, packages]
-    disabled: true
-    timeout: 5.0
-    shortcut: pack
-    about:
-      website: https://packagist.org
-      wikidata_id: Q108311377
-      official_api_documentation: https://packagist.org/apidoc
-      use_official_api: true
-      require_api_key: false
-      results: JSON
-
-  - name: pdbe
-    engine: pdbe
-    shortcut: pdb
-    # Hide obsolete PDB entries.  Default is not to hide obsolete structures
-    #  hide_obsolete: false
-
-  - name: photon
-    engine: photon
-    shortcut: ph
-
-  - name: pinterest
-    engine: pinterest
-    shortcut: pin
-
-  - name: piped
-    engine: piped
-    shortcut: ppd
-    categories: videos
-    piped_filter: videos
-    timeout: 3.0
-
-    # URL to use as link and for embeds
-    frontend_url: https://srv.piped.video
-    # Instance will be selected randomly, for more see https://piped-instances.kavin.rocks/
-    backend_url:
-      - https://pipedapi.adminforge.de
-      - https://pipedapi.nosebs.ru
-      - https://pipedapi.ducks.party
-      - https://pipedapi.reallyaweso.me
-      - https://api.piped.private.coffee
-      - https://pipedapi.darkness.services
-
-  - name: piped.music
-    engine: piped
-    network: piped
-    shortcut: ppdm
-    categories: music
-    piped_filter: music_songs
-    timeout: 3.0
-
-  - name: piratebay
-    engine: piratebay
-    shortcut: tpb
-    # You may need to change this URL to a proxy if piratebay is blocked in your
-    # country
-    url: https://thepiratebay.org/
-    timeout: 3.0
-
-  - name: pixabay images
-    engine: pixabay
-    pixabay_type: images
-    categories: images
-    shortcut: pixi
-    disabled: true
-
-  - name: pixabay videos
-    engine: pixabay
-    pixabay_type: videos
-    categories: videos
-    shortcut: pixv
-    disabled: true
-
-  - name: pixiv
-    shortcut: pv
-    engine: pixiv
-    disabled: true
-    inactive: true
-    pixiv_image_proxies:
-      - https://pximg.example.org
-      # A proxy is required to load the images. Hosting an image proxy server
-      # for Pixiv:
-      #    --> https://pixivfe.pages.dev/hosting-image-proxy-server/
-      # Proxies from public instances.  Ask the public instances owners if they
-      # agree to receive traffic from SearXNG!
-      #    --> https://codeberg.org/VnPower/PixivFE#instances
-      #    --> https://github.com/searxng/searxng/pull/3192#issuecomment-1941095047
-      # image proxy of https://pixiv.cat
-      # - https://i.pixiv.cat
-      # image proxy of https://www.pixiv.pics
-      # - https://pximg.cocomi.eu.org
-      # image proxy of https://pixivfe.exozy.me
-      # - https://pximg.exozy.me
-      # image proxy of https://pixivfe.ducks.party
-      # - https://pixiv.ducks.party
-      # image proxy of https://pixiv.perennialte.ch
-      # - https://pximg.perennialte.ch
-
-  - name: podcastindex
-    engine: podcastindex
-    shortcut: podcast
-
-  # Required dependency: psychopg2
-  #  - name: postgresql
-  #    engine: postgresql
-  #    database: postgres
-  #    username: postgres
-  #    password: postgres
-  #    limit: 10
-  #    query_str: 'SELECT * from my_table WHERE my_column = %(query)s'
-  #    shortcut : psql
-
-  - name: presearch
-    engine: presearch
-    search_type: search
-    categories: [general, web]
-    shortcut: ps
-    timeout: 4.0
-    disabled: true
-
-  - name: presearch images
-    engine: presearch
-    network: presearch
-    search_type: images
-    categories: [images, web]
-    timeout: 4.0
-    shortcut: psimg
-    disabled: true
-
-  - name: presearch videos
-    engine: presearch
-    network: presearch
-    search_type: videos
-    categories: [general, web]
-    timeout: 4.0
-    shortcut: psvid
-    disabled: true
-
-  - name: presearch news
-    engine: presearch
-    network: presearch
-    search_type: news
-    categories: [news, web]
-    timeout: 4.0
-    shortcut: psnews
-    disabled: true
-
-  - name: pub.dev
-    engine: xpath
-    shortcut: pd
-    search_url: https://pub.dev/packages?q={query}&page={pageno}
-    paging: true
-    results_xpath: //div[contains(@class,"packages-item")]
-    url_xpath: ./div/h3/a/@href
-    title_xpath: ./div/h3/a
-    content_xpath: ./div/div/div[contains(@class,"packages-description")]/span
-    categories: [packages, it]
-    timeout: 3.0
-    disabled: true
-    first_page_num: 1
-    about:
-      website: https://pub.dev/
-      official_api_documentation: https://pub.dev/help/api
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: public domain image archive
-    engine: public_domain_image_archive
-    shortcut: pdia
-
-  - name: pubmed
-    engine: pubmed
-    shortcut: pub
-    timeout: 3.0
-
-  - name: pypi
-    shortcut: pypi
-    engine: pypi
-
-  - name: quark
-    quark_category: general
-    categories: [general]
-    engine: quark
-    shortcut: qk
-    disabled: true
-
-  - name: quark images
-    quark_category: images
-    categories: [images]
-    engine: quark
-    shortcut: qki
-    disabled: true
-
-  - name: qwant
-    qwant_categ: web
-    engine: qwant
-    shortcut: qw
-    categories: [general, web]
-    disabled: true
-    additional_tests:
-      rosebud: *test_rosebud
-
-  - name: qwant news
-    qwant_categ: news
-    engine: qwant
-    shortcut: qwn
-    categories: news
-    network: qwant
-
-  - name: qwant images
-    qwant_categ: images
-    engine: qwant
-    shortcut: qwi
-    categories: [images, web]
-    network: qwant
-
-  - name: qwant videos
-    qwant_categ: videos
-    engine: qwant
-    shortcut: qwv
-    categories: [videos, web]
-    network: qwant
-
-  # - name: library
-  #   engine: recoll
-  #   shortcut: lib
-  #   base_url: 'https://recoll.example.org/'
-  #   search_dir: ''
-  #   mount_prefix: /export
-  #   dl_prefix: 'https://download.example.org'
-  #   timeout: 30.0
-  #   categories: files
-  #   disabled: true
-
-  # - name: recoll library reference
-  #   engine: recoll
-  #   base_url: 'https://recoll.example.org/'
-  #   search_dir: reference
-  #   mount_prefix: /export
-  #   dl_prefix: 'https://download.example.org'
-  #   shortcut: libr
-  #   timeout: 30.0
-  #   categories: files
-  #   disabled: true
-
-  - name: radio browser
-    engine: radio_browser
-    shortcut: rb
-
-  - name: reddit
-    engine: reddit
-    shortcut: re
-    page_size: 25
-    disabled: true
-
-  - name: reuters
-    engine: reuters
-    shortcut: reu
-    # https://docs.searxng.org/dev/engines/online/reuters.html
-    # sort_order = "relevance"
-
-  - name: right dao
-    engine: xpath
-    paging: true
-    page_size: 12
-    search_url: https://rightdao.com/search?q={query}&start={pageno}
-    results_xpath: //div[contains(@class, "description")]
-    url_xpath: ../div[contains(@class, "title")]/a/@href
-    title_xpath: ../div[contains(@class, "title")]
-    content_xpath: .
-    categories: general
-    shortcut: rd
-    disabled: true
-    about:
-      website: https://rightdao.com/
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: rottentomatoes
-    engine: rottentomatoes
-    shortcut: rt
-    disabled: true
-
-  # Required dependency: valkey
-  # - name: myvalkey
-  #   shortcut : rds
-  #   engine: valkey_server
-  #   exact_match_only: false
-  #   host: '127.0.0.1'
-  #   port: 6379
-  #   enable_http: true
-  #   password: ''
-  #   db: 0
-
-  # tmp suspended: bad certificate
-  #  - name: scanr structures
-  #    shortcut: scs
-  #    engine: scanr_structures
-  #    disabled: true
-
-  - name: searchmysite
-    engine: xpath
-    shortcut: sms
-    categories: general
-    paging: true
-    search_url: https://searchmysite.net/search/?q={query}&page={pageno}
-    results_xpath: //div[contains(@class,'search-result')]
-    url_xpath: .//a[contains(@class,'result-link')]/@href
-    title_xpath: .//span[contains(@class,'result-title-txt')]/text()
-    content_xpath: ./p[@id='result-hightlight']
-    disabled: true
-    about:
-      website: https://searchmysite.net
-
-  - name: selfhst icons
-    engine: selfhst
-    shortcut: si
-    disabled: true
-
-  - name: sepiasearch
-    engine: sepiasearch
-    shortcut: sep
-
-  - name: sogou
-    engine: sogou
-    shortcut: sogou
-    disabled: true
-
-  - name: sogou images
-    engine: sogou_images
-    shortcut: sogoui
-    disabled: true
-
-  - name: sogou videos
-    engine: sogou_videos
-    shortcut: sogouv
-    disabled: true
-
-  - name: sogou wechat
-    engine: sogou_wechat
-    shortcut: sogouw
-    disabled: true
-
-  - name: soundcloud
-    engine: soundcloud
-    shortcut: sc
-
-  - name: stackoverflow
-    engine: stackexchange
-    shortcut: st
-    api_site: 'stackoverflow'
-    categories: [it, q&a]
-
-  - name: askubuntu
-    engine: stackexchange
-    shortcut: ubuntu
-    api_site: 'askubuntu'
-    categories: [it, q&a]
-
-  - name: superuser
-    engine: stackexchange
-    shortcut: su
-    api_site: 'superuser'
-    categories: [it, q&a]
-
-  - name: discuss.python
-    engine: discourse
-    shortcut: dpy
-    base_url: 'https://discuss.python.org'
-    categories: [it, q&a]
-    disabled: true
-
-  - name: caddy.community
-    engine: discourse
-    shortcut: caddy
-    base_url: 'https://caddy.community'
-    categories: [it, q&a]
-    disabled: true
-
-  - name: pi-hole.community
-    engine: discourse
-    shortcut: pi
-    categories: [it, q&a]
-    base_url: 'https://discourse.pi-hole.net'
-    disabled: true
-
-  - name: searchcode code
-    engine: searchcode_code
-    shortcut: scc
-    disabled: true
-    inactive: true
-
-  # - name: searx
-  #   engine: searx_engine
-  #   shortcut: se
-  #   instance_urls :
-  #       - http://127.0.0.1:8888/
-  #       - ...
-  #   disabled: true
-
-  - name: semantic scholar
-    engine: semantic_scholar
-    disabled: true
-    shortcut: se
-
-  # Spotify needs API credentials
-  # - name: spotify
-  #   engine: spotify
-  #   shortcut: stf
-  #   api_client_id: *******
-  #   api_client_secret: *******
-
-  # - name: solr
-  #   engine: solr
-  #   shortcut: slr
-  #   base_url: http://localhost:8983
-  #   collection: collection_name
-  #   sort: '' # sorting: asc or desc
-  #   field_list: '' # comma separated list of field names to display on the UI
-  #   default_fields: '' # default field to query
-  #   query_fields: '' # query fields
-  #   enable_http: true
-
-  # - name: springer nature
-  #   engine: springer
-  #   # get your API key from: https://dev.springernature.com/signup
-  #   # working API key, for test & debug: "a69685087d07eca9f13db62f65b8f601"
-  #   api_key: 'unset'
-  #   shortcut: springer
-  #   timeout: 15.0
-
-  - name: startpage
-    engine: startpage
-    shortcut: sp
-    startpage_categ: web
-    categories: [general, web]
-    additional_tests:
-      rosebud: *test_rosebud
-
-  - name: startpage news
-    engine: startpage
-    startpage_categ: news
-    categories: [news, web]
-    shortcut: spn
-
-  - name: startpage images
-    engine: startpage
-    startpage_categ: images
-    categories: [images, web]
-    shortcut: spi
-
-  - name: steam
-    engine: steam
-    shortcut: stm
-    disabled: true
-
-  - name: tokyotoshokan
-    engine: tokyotoshokan
-    shortcut: tt
-    timeout: 6.0
-    disabled: true
-
-  - name: solidtorrents
-    engine: solidtorrents
-    shortcut: solid
-    timeout: 4.0
-    base_url:
-      - https://solidtorrents.to
-      - https://bitsearch.to
-
-  # For this demo of the sqlite engine download:
-  #   https://liste.mediathekview.de/filmliste-v2.db.bz2
-  # and unpack into searx/data/filmliste-v2.db
-  # Query to test: "!mediathekview concert"
-  #
-  # - name: mediathekview
-  #   engine: sqlite
-  #   shortcut: mediathekview
-  #   categories: [general, videos]
-  #   result_type: MainResult
-  #   database: searx/data/filmliste-v2.db
-  #   query_str: >-
-  #     SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
-  #            COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
-  #            description AS content
-  #       FROM film
-  #      WHERE title LIKE :wildcard OR description LIKE :wildcard
-  #      ORDER BY duration DESC
-
-  - name: tagesschau
-    engine: tagesschau
-    # when set to false, display URLs from Tagesschau, and not the actual source
-    # (e.g. NDR, WDR, SWR, HR, ...)
-    use_source_url: true
-    shortcut: ts
-    disabled: true
-
-  - name: tmdb
-    engine: xpath
-    paging: true
-    categories: movies
-    search_url: https://www.themoviedb.org/search?page={pageno}&query={query}
-    results_xpath: //div[contains(@class,"movie") or contains(@class,"tv")]//div[contains(@class,"card")]
-    url_xpath: .//div[contains(@class,"poster")]/a/@href
-    thumbnail_xpath: .//img/@src
-    title_xpath: .//div[contains(@class,"title")]//h2
-    content_xpath: .//div[contains(@class,"overview")]
-    shortcut: tm
-    disabled: true
-
-  # Requires Tor
-  - name: torch
-    engine: xpath
-    paging: true
-    search_url:
-      http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega?P={query}&DEFAULTOP=and
-    results_xpath: //table//tr
-    url_xpath: ./td[2]/a
-    title_xpath: ./td[2]/b
-    content_xpath: ./td[2]/small
-    categories: onions
-    enable_http: true
-    shortcut: tch
-
-  # TubeArchivist is a self-hosted Youtube archivist software.
-  # https://docs.searxng.org/dev/engines/online/tubearchivist.html
-  #
-  # - name: tubearchivist
-  #   engine: tubearchivist
-  #   shortcut: tuba
-  #   base_url:
-  #   ta_token:
-  #   ta_link_to_mp4: false
-
-  # torznab engine lets you query any torznab compatible indexer.  Using this
-  # engine in combination with Jackett opens the possibility to query a lot of
-  # public and private indexers directly from SearXNG. More details at:
-  # https://docs.searxng.org/dev/engines/online/torznab.html
-  #
-  # - name: Torznab EZTV
-  #   engine: torznab
-  #   shortcut: eztv
-  #   base_url: http://localhost:9117/api/v2.0/indexers/eztv/results/torznab
-  #   enable_http: true  # if using localhost
-  #   api_key: xxxxxxxxxxxxxxx
-  #   show_magnet_links: true
-  #   show_torrent_files: false
-  #   # https://github.com/Jackett/Jackett/wiki/Jackett-Categories
-  #   torznab_categories:  # optional
-  #     - 2000
-  #     - 5000
-
-  # tmp suspended - too slow, too many errors
-  #  - name: urbandictionary
-  #    engine      : xpath
-  #    search_url  : https://www.urbandictionary.com/define.php?term={query}
-  #    url_xpath   : //*[@class="word"]/@href
-  #    title_xpath : //*[@class="def-header"]
-  #    content_xpath: //*[@class="meaning"]
-  #    shortcut: ud
-
-  - name: unsplash
-    engine: unsplash
-    shortcut: us
-
-  - name: yandex
-    engine: yandex
-    categories: general
-    search_type: web
-    shortcut: yd
-    disabled: true
-    inactive: true
-
-  - name: yandex images
-    engine: yandex
-    categories: images
-    search_type: images
-    shortcut: ydi
-    disabled: true
-    inactive: true
-
-  - name: yandex music
-    engine: yandex_music
-    shortcut: ydm
-    disabled: true
-    # https://yandex.com/support/music/access.html
-    inactive: true
-
-  - name: yahoo
-    engine: yahoo
-    shortcut: yh
-    disabled: true
-
-  - name: yahoo news
-    engine: yahoo_news
-    shortcut: yhn
-
-  - name: youtube
-    shortcut: yt
-    # You can use the engine using the official stable API, but you need an API
-    # key See: https://console.developers.google.com/project
-    #
-    # engine: youtube_api
-    # api_key: 'apikey' # required!
-    #
-    # Or you can use the html non-stable engine, activated by default
-    engine: youtube_noapi
-
-  - name: dailymotion
-    engine: dailymotion
-    shortcut: dm
-
-  - name: vimeo
-    engine: vimeo
-    shortcut: vm
-
-  - name: wiby
-    engine: json_engine
-    paging: true
-    search_url: https://wiby.me/json/?q={query}&p={pageno}
-    url_query: URL
-    title_query: Title
-    content_query: Snippet
-    categories: [general, web]
-    shortcut: wib
-    disabled: true
-    about:
-      website: https://wiby.me/
-
-  - name: wikibooks
-    engine: mediawiki
-    weight: 0.5
-    shortcut: wb
-    categories: [general, wikimedia]
-    base_url: "https://{language}.wikibooks.org/"
-    search_type: text
-    disabled: true
-    about:
-      website: https://www.wikibooks.org/
-      wikidata_id: Q367
-
-  - name: wikinews
-    engine: mediawiki
-    shortcut: wn
-    categories: [news, wikimedia]
-    base_url: "https://{language}.wikinews.org/"
-    search_type: text
-    srsort: create_timestamp_desc
-    about:
-      website: https://www.wikinews.org/
-      wikidata_id: Q964
-
-  - name: wikiquote
-    engine: mediawiki
-    weight: 0.5
-    shortcut: wq
-    categories: [general, wikimedia]
-    base_url: "https://{language}.wikiquote.org/"
-    search_type: text
-    disabled: true
-    additional_tests:
-      rosebud: *test_rosebud
-    about:
-      website: https://www.wikiquote.org/
-      wikidata_id: Q369
-
-  - name: wikisource
-    engine: mediawiki
-    weight: 0.5
-    shortcut: ws
-    categories: [general, wikimedia]
-    base_url: "https://{language}.wikisource.org/"
-    search_type: text
-    disabled: true
-    about:
-      website: https://www.wikisource.org/
-      wikidata_id: Q263
-
-  - name: wikispecies
-    engine: mediawiki
-    shortcut: wsp
-    categories: [general, science, wikimedia]
-    base_url: "https://species.wikimedia.org/"
-    search_type: text
-    disabled: true
-    about:
-      website: https://species.wikimedia.org/
-      wikidata_id: Q13679
-    tests:
-      wikispecies:
-        matrix:
-          query: "Campbell, L.I. et al. 2011: MicroRNAs"
-          lang: en
-        result_container:
-          - not_empty
-          - ['one_title_contains', 'Tardigrada']
-        test:
-          - unique_results
-
-  - name: wiktionary
-    engine: mediawiki
-    shortcut: wt
-    categories: [dictionaries, wikimedia]
-    base_url: "https://{language}.wiktionary.org/"
-    search_type: text
-    about:
-      website: https://www.wiktionary.org/
-      wikidata_id: Q151
-
-  - name: wikiversity
-    engine: mediawiki
-    weight: 0.5
-    shortcut: wv
-    categories: [general, wikimedia]
-    base_url: "https://{language}.wikiversity.org/"
-    search_type: text
-    disabled: true
-    about:
-      website: https://www.wikiversity.org/
-      wikidata_id: Q370
-
-  - name: wikivoyage
-    engine: mediawiki
-    weight: 0.5
-    shortcut: wy
-    categories: [general, wikimedia]
-    base_url: "https://{language}.wikivoyage.org/"
-    search_type: text
-    disabled: true
-    about:
-      website: https://www.wikivoyage.org/
-      wikidata_id: Q373
-
-  - name: wikicommons.images
-    engine: wikicommons
-    shortcut: wc
-    categories: images
-    search_type: images
-    number_of_results: 10
-
-  - name: wikicommons.videos
-    engine: wikicommons
-    shortcut: wcv
-    categories: videos
-    search_type: videos
-    number_of_results: 10
-
-  - name: wikicommons.audio
-    engine: wikicommons
-    shortcut: wca
-    categories: music
-    search_type: audio
-    number_of_results: 10
-
-  - name: wikicommons.files
-    engine: wikicommons
-    shortcut: wcf
-    categories: files
-    search_type: files
-    number_of_results: 10
-
-  - name: wolframalpha
-    shortcut: wa
-    # You can use the engine using the official stable API, but you need an API
-    # key.  See: https://products.wolframalpha.com/api/
-    #
-    # engine: wolframalpha_api
-    # api_key: ''
-    #
-    # Or you can use the html non-stable engine, activated by default
-    engine: wolframalpha_noapi
-    timeout: 6.0
-    categories: general
-    disabled: true
-
-  - name: dictzone
-    engine: dictzone
-    shortcut: dc
-
-  - name: mymemory translated
-    engine: translated
-    shortcut: tl
-    timeout: 5.0
-    # You can use without an API key, but you are limited to 1000 words/day
-    # See: https://mymemory.translated.net/doc/usagelimits.php
-    # api_key: ''
-
-  # Required dependency: mysql-connector-python
-  #  - name: mysql
-  #    engine: mysql_server
-  #    database: mydatabase
-  #    username: user
-  #    password: pass
-  #    limit: 10
-  #    query_str: 'SELECT * from mytable WHERE fieldname=%(query)s'
-  #    shortcut: mysql
-
-  # Required dependency: mariadb
-  #  - name: mariadb
-  #    engine: mariadb_server
-  #    database: mydatabase
-  #    username: user
-  #    password: pass
-  #    limit: 10
-  #    query_str: 'SELECT * from mytable WHERE fieldname=%(query)s'
-  #    shortcut: mdb
-
-  - name: 1337x
-    engine: 1337x
-    shortcut: 1337x
-    disabled: true
-
-  - name: duden
-    engine: duden
-    shortcut: du
-    disabled: true
-
-  - name: seznam
-    shortcut: szn
-    engine: seznam
-    disabled: true
-
-  # - name: deepl
-  #   engine: deepl
-  #   shortcut: dpl
-  #   # You can use the engine using the official stable API, but you need an API key
-  #   # See: https://www.deepl.com/pro-api?cta=header-pro-api
-  #   api_key: ''  # required!
-  #   timeout: 5.0
-  #   disabled: true
-
-  - name: mojeek
-    shortcut: mjk
-    engine: mojeek
-    categories: [general, web]
-    disabled: true
-
-  - name: mojeek images
-    shortcut: mjkimg
-    engine: mojeek
-    categories: [images, web]
-    search_type: images
-    paging: false
-    disabled: true
-
-  - name: mojeek news
-    shortcut: mjknews
-    engine: mojeek
-    categories: [news, web]
-    search_type: news
-    paging: false
-    disabled: true
-
-  - name: moviepilot
-    engine: moviepilot
-    shortcut: mp
-    disabled: true
-
-  - name: naver
-    categories: [general, web]
-    engine: naver
-    shortcut: nvr
-    disabled: true
-
-  - name: naver images
-    naver_category: images
-    categories: [images]
-    engine: naver
-    shortcut: nvri
-    disabled: true
-
-  - name: naver news
-    naver_category: news
-    categories: [news]
-    engine: naver
-    shortcut: nvrn
-    disabled: true
-
-  - name: naver videos
-    naver_category: videos
-    categories: [videos]
-    engine: naver
-    shortcut: nvrv
-    disabled: true
-
-  - name: rubygems
-    shortcut: rbg
-    engine: xpath
-    paging: true
-    search_url: https://rubygems.org/search?page={pageno}&query={query}
-    results_xpath: /html/body/main/div/a[@class="gems__gem"]
-    url_xpath: ./@href
-    title_xpath: ./span/h2
-    content_xpath: ./span/p
-    suggestion_xpath: /html/body/main/div/div[@class="search__suggestions"]/p/a
-    first_page_num: 1
-    categories: [it, packages]
-    disabled: true
-    about:
-      website: https://rubygems.org/
-      wikidata_id: Q1853420
-      official_api_documentation: https://guides.rubygems.org/rubygems-org-api/
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: peertube
-    engine: peertube
-    shortcut: ptb
-    paging: true
-    # alternatives see: https://instances.joinpeertube.org/instances
-    # base_url: https://tube.4aem.com
-    categories: videos
-    disabled: true
-    timeout: 6.0
-
-  - name: mediathekviewweb
-    engine: mediathekviewweb
-    shortcut: mvw
-    disabled: true
-
-  - name: yacy
-    # https://docs.searxng.org/dev/engines/online/yacy.html
-    engine: yacy
-    categories: general
-    search_type: text
-    base_url:
-      - https://yacy.searchlab.eu
-      # see https://github.com/searxng/searxng/pull/3631#issuecomment-2240903027
-      # - https://search.kyun.li
-      # - https://yacy.securecomcorp.eu
-      # - https://yacy.myserv.ca
-      # - https://yacy.nsupdate.info
-      # - https://yacy.electroncash.de
-    shortcut: ya
-    disabled: true
-    # if you aren't using HTTPS for your local yacy instance disable https
-    # enable_http: false
-    search_mode: 'global'
-    # timeout can be reduced in 'local' search mode
-    timeout: 5.0
-
-  - name: yacy images
-    engine: yacy
-    network: yacy
-    categories: images
-    search_type: image
-    shortcut: yai
-    disabled: true
-    # timeout can be reduced in 'local' search mode
-    timeout: 5.0
-
-  - name: rumble
-    engine: rumble
-    shortcut: ru
-    base_url: https://rumble.com/
-    paging: true
-    categories: videos
-    disabled: true
-
-  - name: repology
-    engine: repology
-    shortcut: rep
-    disabled: true
-    inactive: true
-
-  - name: livespace
-    engine: livespace
-    shortcut: ls
-    categories: videos
-    disabled: true
-    timeout: 5.0
-
-  - name: wordnik
-    engine: wordnik
-    shortcut: wnik
-    timeout: 5.0
-
-  - name: woxikon.de synonyme
-    engine: xpath
-    shortcut: woxi
-    categories: [dictionaries]
-    timeout: 5.0
-    disabled: true
-    search_url: https://synonyme.woxikon.de/synonyme/{query}.php
-    url_xpath: //div[@class="upper-synonyms"]/a/@href
-    content_xpath: //div[@class="synonyms-list-group"]
-    title_xpath: //div[@class="upper-synonyms"]/a
-    no_result_for_http_status: [404]
-    about:
-      website: https://www.woxikon.de/
-      wikidata_id:  # No Wikidata ID
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-      language: de
-
-  - name: seekr news
-    engine: seekr
-    shortcut: senews
-    categories: news
-    seekr_category: news
-    disabled: true
-
-  - name: seekr images
-    engine: seekr
-    network: seekr news
-    shortcut: seimg
-    categories: images
-    seekr_category: images
-    disabled: true
-
-  - name: seekr videos
-    engine: seekr
-    network: seekr news
-    shortcut: sevid
-    categories: videos
-    seekr_category: videos
-    disabled: true
-
-  - name: stract
-    engine: stract
-    shortcut: str
-    disabled: true
-
-  - name: svgrepo
-    engine: svgrepo
-    shortcut: svg
-    timeout: 10.0
-    disabled: true
-
-  - name: tootfinder
-    engine: tootfinder
-    shortcut: toot
-
-  - name: uxwing
-    engine: uxwing
-    shortcut: ux
-    disabled: true
-
-  - name: voidlinux
-    engine: voidlinux
-    shortcut: void
-    disabled: true
-
-  - name: wallhaven
-    engine: wallhaven
-    # api_key: abcdefghijklmnopqrstuvwxyz
-    shortcut: wh
-    disabled: true
-
-    # wikimini: online encyclopedia for children
-    # The fulltext and title parameter is necessary for Wikimini because
-    # sometimes it will not show the results and redirect instead
-  - name: wikimini
-    engine: xpath
-    shortcut: wkmn
-    search_url: https://fr.wikimini.org/w/index.php?search={query}&title=Sp%C3%A9cial%3ASearch&fulltext=Search
-    url_xpath: //li/div[@class="mw-search-result-heading"]/a/@href
-    title_xpath: //li//div[@class="mw-search-result-heading"]/a
-    content_xpath: //li/div[@class="searchresult"]
-    categories: general
-    disabled: true
-    about:
-      website: https://wikimini.org/
-      wikidata_id: Q3568032
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-      language: fr
-
-  - name: wttr.in
-    engine: wttr
-    shortcut: wttr
-    timeout: 9.0
-
-  - name: brave
-    engine: brave
-    shortcut: br
-    time_range_support: true
-    paging: true
-    categories: [general, web]
-    brave_category: search
-    # brave_spellcheck: true
-
-  - name: brave.images
-    engine: brave
-    network: brave
-    shortcut: brimg
-    categories: [images, web]
-    brave_category: images
-
-  - name: brave.videos
-    engine: brave
-    network: brave
-    shortcut: brvid
-    categories: [videos, web]
-    brave_category: videos
-
-  - name: brave.news
-    engine: brave
-    network: brave
-    shortcut: brnews
-    categories: news
-    brave_category: news
-
-  # - name: brave.goggles
-  #   engine: brave
-  #   network: brave
-  #   shortcut: brgog
-  #   time_range_support: true
-  #   paging: true
-  #   categories: [general, web]
-  #   brave_category: goggles
-  #   Goggles: # required! This should be a URL ending in .goggle
-
-  - name: lib.rs
-    shortcut: lrs
-    engine: lib_rs
-    disabled: true
-
-  - name: sourcehut
-    shortcut: srht
-    engine: xpath
-    paging: true
-    search_url: https://sr.ht/projects?page={pageno}&search={query}
-    results_xpath: (//div[@class="event-list"])[1]/div[@class="event"]
-    url_xpath: ./h4/a[2]/@href
-    title_xpath: ./h4/a[2]
-    content_xpath: ./p
-    first_page_num: 1
-    categories: [it, repos]
-    disabled: true
-    about:
-      website: https://sr.ht
-      wikidata_id: Q78514485
-      official_api_documentation: https://man.sr.ht/
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
-  - name: goo
-    shortcut: goo
-    engine: xpath
-    paging: true
-    search_url: https://search.goo.ne.jp/web.jsp?MT={query}&FR={pageno}0
-    url_xpath: //div[@class="result"]/p[@class='title fsL1']/a/@href
-    title_xpath: //div[@class="result"]/p[@class='title fsL1']/a
-    content_xpath: //p[contains(@class,'url fsM')]/following-sibling::p
-    first_page_num: 0
-    categories: [general, web]
-    disabled: true
-    timeout: 4.0
-    about:
-      website: https://search.goo.ne.jp
-      wikidata_id: Q249044
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-      language: ja
-
-  - name: bt4g
-    engine: bt4g
-    shortcut: bt4g
-
-  - name: pkg.go.dev
-    engine: pkg_go_dev
-    shortcut: pgo
-    disabled: true
-
-  - name: senscritique
-    engine: senscritique
-    shortcut: scr
-    timeout: 4.0
-    disabled: true
-
-  - name: minecraft wiki
-    engine: mediawiki
-    shortcut: mcw
-    categories: ["software wikis"]
-    base_url: https://minecraft.wiki/
-    api_path: "api.php"
-    search_type: text
-    disabled: true
-    about:
-      website: https://minecraft.wiki/
-      wikidata_id: Q105533483
-
-# Doku engine lets you access to any Doku wiki instance:
-# A public one or a privete/corporate one.
-#  - name: ubuntuwiki
-#    engine: doku
-#    shortcut: uw
-#    base_url: 'https://doc.ubuntu-fr.org'
-
-# Be careful when enabling this engine if you are
-# running a public instance. Do not expose any sensitive
-# information. You can restrict access by configuring a list
-# of access tokens under tokens.
-#  - name: git grep
-#    engine: command
-#    command: ['git', 'grep', '{{QUERY}}']
-#    shortcut: gg
-#    tokens: []
-#    disabled: true
-#    delimiter:
-#        chars: ':'
-#        keys: ['filepath', 'code']
-
-# Be careful when enabling this engine if you are
-# running a public instance. Do not expose any sensitive
-# information. You can restrict access by configuring a list
-# of access tokens under tokens.
-#  - name: locate
-#    engine: command
-#    command: ['locate', '{{QUERY}}']
-#    shortcut: loc
-#    tokens: []
-#    disabled: true
-#    delimiter:
-#        chars: ' '
-#        keys: ['line']
-
-# Be careful when enabling this engine if you are
-# running a public instance. Do not expose any sensitive
-# information. You can restrict access by configuring a list
-# of access tokens under tokens.
-#  - name: find
-#    engine: command
-#    command: ['find', '.', '-name', '{{QUERY}}']
-#    query_type: path
-#    shortcut: fnd
-#    tokens: []
-#    disabled: true
-#    delimiter:
-#        chars: ' '
-#        keys: ['line']
-
-# Be careful when enabling this engine if you are
-# running a public instance. Do not expose any sensitive
-# information. You can restrict access by configuring a list
-# of access tokens under tokens.
-#  - name: pattern search in files
-#    engine: command
-#    command: ['fgrep', '{{QUERY}}']
-#    shortcut: fgr
-#    tokens: []
-#    disabled: true
-#    delimiter:
-#        chars: ' '
-#        keys: ['line']
-
-# Be careful when enabling this engine if you are
-# running a public instance. Do not expose any sensitive
-# information. You can restrict access by configuring a list
-# of access tokens under tokens.
-#  - name: regex search in files
-#    engine: command
-#    command: ['grep', '{{QUERY}}']
-#    shortcut: gr
-#    tokens: []
-#    disabled: true
-#    delimiter:
-#        chars: ' '
-#        keys: ['line']
-
-doi_resolvers:
-  oadoi.org: 'https://oadoi.org/'
-  doi.org: 'https://doi.org/'
-  sci-hub.se: 'https://sci-hub.se/'
-  sci-hub.st: 'https://sci-hub.st/'
-  sci-hub.ru: 'https://sci-hub.ru/'
-
-default_doi_resolver: 'oadoi.org'
diff --git a/apps/backend/test_advanced_integration.py b/apps/backend/test_advanced_integration.py
deleted file mode 100644
index 88e86a7..0000000
--- a/apps/backend/test_advanced_integration.py
+++ /dev/null
@@ -1,324 +0,0 @@
-"""
-Integration tests for advanced Firecrawl-inspired features.
-
-Tests all the newly implemented advanced functionalities:
-- Multi-provider search integration
-- Multi-engine scraping architecture  
-- LLM-powered configuration generation
-- Advanced batch processing operations
-- Multi-entity extraction service
-"""
-
-import asyncio
-import json
-import pytest
-from typing import Dict, Any, List
-
-# Test individual services
-async def test_multi_search_service():
-    """Test multi-provider search service."""
-    try:
-        from app.services.multi_search import get_multi_search_service, SearchOptions
-        
-        print("🔍 Testing Multi-Provider Search Service...")
-        
-        service = await get_multi_search_service()
-        
-        # Test basic search
-        options = SearchOptions(
-            query="artificial intelligence",
-            num_results=5,
-            lang="en",
-            country="us"
-        )
-        
-        results = await service.search(options)
-        print(f"✅ Search completed: {len(results)} results found")
-        
-        # Test provider stats
-        stats = await service.get_provider_stats()
-        print(f"✅ Provider stats retrieved: {len(stats['providers'])} providers configured")
-        
-        return True
-        
-    except Exception as e:
-        print(f"❌ Multi-search service test failed: {str(e)}")
-        return False
-
-
-async def test_multi_engine_scraping():
-    """Test multi-engine scraping service."""
-    try:
-        from app.services.multi_engine_scraper import get_multi_engine_service
-        from app.models.requests import ScrapingConfig
-        
-        print("🤖 Testing Multi-Engine Scraping Service...")
-        
-        service = await get_multi_engine_service()
-        
-        # Test scraping with fallback
-        test_url = "https://example.com"
-        config = ScrapingConfig(urls=[test_url])
-        
-        result = await service.scrape(test_url, config)
-        print(f"✅ Scraping completed: engine={result.engine_used.value}, success={result.success}")
-        
-        # Test engine stats
-        stats = await service.get_engine_stats()
-        print(f"✅ Engine stats retrieved: {stats['total_engines']} engines available")
-        
-        return True
-        
-    except Exception as e:
-        print(f"❌ Multi-engine scraping test failed: {str(e)}")
-        return False
-
-
-async def test_llm_configuration():
-    """Test LLM-powered configuration generation."""
-    try:
-        from app.services.llm_configuration import get_llm_config_service, generate_config_from_prompt
-        
-        print("🧠 Testing LLM Configuration Service...")
-        
-        # Test configuration generation
-        test_prompt = "Crawl a blog website and extract only article pages, excluding navigation and sidebar content"
-        
-        try:
-            config = await generate_config_from_prompt(
-                prompt=test_prompt,
-                config_type="crawler"
-            )
-            print(f"✅ LLM config generation completed: {len(config)} options generated")
-        except Exception as llm_error:
-            print(f"⚠️ LLM config generation skipped (likely no OpenAI key): {str(llm_error)}")
-        
-        # Test service stats
-        service = await get_llm_config_service()
-        stats = await service.get_usage_stats()
-        print(f"✅ LLM service stats retrieved: {stats['usage_stats']['total_requests']} requests processed")
-        
-        return True
-        
-    except Exception as e:
-        print(f"❌ LLM configuration test failed: {str(e)}")
-        return False
-
-
-async def test_batch_operations():
-    """Test batch operation service."""
-    try:
-        from app.services.batch_operations import get_batch_service
-        
-        print("📦 Testing Batch Operations Service...")
-        
-        service = await get_batch_service()
-        
-        # Test batch scraping job submission
-        test_urls = ["https://example.com", "https://httpbin.org/json"]
-        
-        job_id = await service.submit_batch_scrape(
-            urls=test_urls,
-            priority=20,
-            metadata={"test": "integration"}
-        )
-        
-        print(f"✅ Batch job submitted: {job_id}")
-        
-        # Test job status
-        status = await service.get_job_status(job_id)
-        if status:
-            print(f"✅ Job status retrieved: {status.status.value}")
-        
-        # Test service stats
-        stats = await service.get_service_stats()
-        print(f"✅ Batch service stats: {stats['active_jobs']} active jobs")
-        
-        return True
-        
-    except Exception as e:
-        print(f"❌ Batch operations test failed: {str(e)}")
-        return False
-
-
-async def test_multi_entity_extraction():
-    """Test multi-entity extraction service."""
-    try:
-        from app.services.multi_entity_extraction import (
-            get_multi_entity_service, 
-            MultiEntityExtractionRequest,
-            ExtractionStrategy
-        )
-        
-        print("🔗 Testing Multi-Entity Extraction Service...")
-        
-        service = await get_multi_entity_service()
-        
-        # Test extraction request
-        test_schema = {
-            "type": "object",
-            "properties": {
-                "title": {"type": "string"},
-                "description": {"type": "string"}
-            }
-        }
-        
-        request = MultiEntityExtractionRequest(
-            urls=["https://example.com"],
-            schema=test_schema,
-            extraction_strategy=ExtractionStrategy.LINKED_ENTITIES,
-            max_related_urls=10,
-            follow_links=False  # Keep it simple for testing
-        )
-        
-        result = await service.extract_multi_entity(request)
-        print(f"✅ Multi-entity extraction completed: {len(result.entities)} entities, success={result.success}")
-        
-        # Test service stats
-        stats = await service.get_extraction_stats()
-        print(f"✅ Extraction service stats: {stats['extraction_stats']['total_requests']} requests processed")
-        
-        return True
-        
-    except Exception as e:
-        print(f"❌ Multi-entity extraction test failed: {str(e)}")
-        return False
-
-
-async def test_api_endpoints():
-    """Test API endpoints are properly configured."""
-    try:
-        print("🌐 Testing API Endpoint Configuration...")
-        
-        # Test that we can import the routers without errors
-        from app.api.v1.enhanced_search import router as enhanced_router
-        from app.api.v2.advanced_endpoints import router as advanced_router
-        
-        print(f"✅ Enhanced search router: {len(enhanced_router.routes)} routes")
-        print(f"✅ Advanced endpoints router: {len(advanced_router.routes)} routes")
-        
-        # Test that main app includes the routers
-        from app.main import app
-        
-        total_routes = len(app.routes)
-        print(f"✅ Main app configured: {total_routes} total routes")
-        
-        return True
-        
-    except Exception as e:
-        print(f"❌ API endpoints test failed: {str(e)}")
-        return False
-
-
-async def run_integration_tests():
-    """Run all integration tests."""
-    print("🚀 Starting Advanced Features Integration Tests")
-    print("=" * 60)
-    
-    tests = [
-        ("Multi-Provider Search", test_multi_search_service),
-        ("Multi-Engine Scraping", test_multi_engine_scraping),  
-        ("LLM Configuration", test_llm_configuration),
-        ("Batch Operations", test_batch_operations),
-        ("Multi-Entity Extraction", test_multi_entity_extraction),
-        ("API Endpoints", test_api_endpoints)
-    ]
-    
-    results = []
-    
-    for test_name, test_func in tests:
-        print(f"\n📋 Running {test_name} Test...")
-        try:
-            success = await test_func()
-            results.append((test_name, success))
-        except Exception as e:
-            print(f"💥 {test_name} test crashed: {str(e)}")
-            results.append((test_name, False))
-    
-    print("\n" + "=" * 60)
-    print("🎯 INTEGRATION TEST RESULTS")
-    print("=" * 60)
-    
-    passed = 0
-    total = len(results)
-    
-    for test_name, success in results:
-        status = "✅ PASS" if success else "❌ FAIL"
-        print(f"{status} {test_name}")
-        if success:
-            passed += 1
-    
-    print(f"\n📊 Summary: {passed}/{total} tests passed ({passed/total*100:.1f}%)")
-    
-    if passed == total:
-        print("🎉 ALL TESTS PASSED! Advanced features successfully integrated!")
-    else:
-        print(f"⚠️  {total-passed} tests failed. Review the output above for details.")
-    
-    return passed == total
-
-
-async def test_feature_completeness():
-    """Test that all Firecrawl-inspired features are implemented."""
-    print("\n🔍 Testing Feature Completeness...")
-    
-    features_to_check = {
-        "Multi-Provider Search": "app.services.multi_search",
-        "Multi-Engine Scraping": "app.services.multi_engine_scraper", 
-        "LLM Configuration": "app.services.llm_configuration",
-        "Batch Operations": "app.services.batch_operations",
-        "Multi-Entity Extraction": "app.services.multi_entity_extraction",
-        "Enhanced API v1": "app.api.v1.enhanced_search",
-        "Advanced API v2": "app.api.v2.advanced_endpoints"
-    }
-    
-    implemented_features = []
-    missing_features = []
-    
-    for feature_name, module_path in features_to_check.items():
-        try:
-            __import__(module_path)
-            implemented_features.append(feature_name)
-            print(f"✅ {feature_name}")
-        except ImportError as e:
-            missing_features.append((feature_name, str(e)))
-            print(f"❌ {feature_name}: {str(e)}")
-    
-    print(f"\n📈 Feature Implementation: {len(implemented_features)}/{len(features_to_check)} features")
-    
-    if missing_features:
-        print("\n⚠️  Missing Features:")
-        for feature, error in missing_features:
-            print(f"   - {feature}: {error}")
-    else:
-        print("🎉 All advanced features successfully implemented!")
-    
-    return len(missing_features) == 0
-
-
-if __name__ == "__main__":
-    async def main():
-        print("🔥 FIRECRAWL-INSPIRED FEATURES INTEGRATION TEST")
-        print("Testing advanced search and scraping capabilities...")
-        print()
-        
-        # Test feature completeness
-        completeness_ok = await test_feature_completeness()
-        
-        # Run integration tests
-        integration_ok = await run_integration_tests()
-        
-        # Overall result
-        if completeness_ok and integration_ok:
-            print("\n🏆 SUCCESS: All Firecrawl-inspired features successfully integrated!")
-            print("\nYour backend now includes:")
-            print("• Multi-provider search with intelligent fallback")
-            print("• Multi-engine scraping architecture")
-            print("• LLM-powered configuration generation")
-            print("• Advanced batch processing operations")
-            print("• Multi-entity extraction with relationship mapping")
-            print("• Enhanced API endpoints with comprehensive functionality")
-        else:
-            print("\n⚠️ PARTIAL SUCCESS: Some features may need attention")
-    
-    asyncio.run(main())
diff --git a/apps/backend/tests/conftest.py b/apps/backend/tests/conftest.py
deleted file mode 100644
index 19b788c..0000000
--- a/apps/backend/tests/conftest.py
+++ /dev/null
@@ -1,178 +0,0 @@
-"""
-Pytest configuration and fixtures.
-"""
-import asyncio
-import os
-import pytest
-from typing import AsyncGenerator, Generator
-from httpx import AsyncClient
-from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
-from fakeredis import FakeAsyncRedis
-
-from app.main import app
-from app.config import Settings, get_settings
-from app.models.database import Base
-from app.services.database import DatabaseService
-from app.services.cache import CacheService
-from app.services.searxng import SearXNGService
-from app.services.scraping import ContentScrapingService
-
-
-# Test settings
-@pytest.fixture
-def test_settings() -> Settings:
-    """Override settings for testing."""
-    return Settings(
-        environment="testing",
-        database_url=os.getenv("DATABASE_URL", "postgresql://test:test@localhost:5432/test_db"),
-        redis_url=os.getenv("REDIS_URL", "redis://localhost:6379/15"),
-        searxng_url="http://localhost:8888",
-        api_keys=["test-key-1", "test-key-2"],
-        rate_limit_enabled=False,
-        cache_default_ttl=60
-    )
-
-
-@pytest.fixture
-def override_settings(test_settings: Settings):
-    """Override application settings."""
-    app.dependency_overrides[get_settings] = lambda: test_settings
-    yield
-    app.dependency_overrides.clear()
-
-
-# Database fixtures
-@pytest.fixture
-async def test_db(test_settings: Settings) -> AsyncGenerator[DatabaseService, None]:
-    """Create test database."""
-    # Create test engine
-    engine = create_async_engine(
-        str(test_settings.database_url),
-        echo=False,
-        future=True
-    )
-    
-    # Create tables
-    async with engine.begin() as conn:
-        await conn.run_sync(Base.metadata.create_all)
-    
-    # Create service
-    db_service = DatabaseService()
-    db_service.engine = engine
-    db_service.async_session = async_sessionmaker(
-        engine,
-        class_=AsyncSession,
-        expire_on_commit=False
-    )
-    
-    yield db_service
-    
-    # Cleanup
-    await engine.dispose()
-
-
-# Cache fixtures
-@pytest.fixture
-async def test_cache() -> AsyncGenerator[CacheService, None]:
-    """Create test cache service with fake Redis."""
-    cache_service = CacheService()
-    cache_service._client = FakeAsyncRedis()
-    
-    yield cache_service
-    
-    await cache_service.close()
-
-
-# HTTP client fixtures
-@pytest.fixture
-async def client(override_settings) -> AsyncGenerator[AsyncClient, None]:
-    """Create test HTTP client."""
-    async with AsyncClient(app=app, base_url="http://test") as ac:
-        yield ac
-
-
-@pytest.fixture
-async def authenticated_client(client: AsyncClient) -> AsyncClient:
-    """Create authenticated test client."""
-    client.headers["X-API-Key"] = "test-key-1"
-    return client
-
-
-# Mock service fixtures
-@pytest.fixture
-def mock_searxng(mocker):
-    """Mock SearXNG service."""
-    mock = mocker.Mock(spec=SearXNGService)
-    mock.search.return_value = []
-    mock.get_available_engines.return_value = {}
-    mock.health_check.return_value = {
-        "status": "healthy",
-        "latency_ms": 100
-    }
-    return mock
-
-
-@pytest.fixture
-def mock_scraper(mocker):
-    """Mock scraping service."""
-    mock = mocker.Mock(spec=ContentScrapingService)
-    mock.scrape_urls.return_value = []
-    return mock
-
-
-# Event loop configuration
-@pytest.fixture(scope="session")
-def event_loop():
-    """Create event loop for async tests."""
-    loop = asyncio.get_event_loop_policy().new_event_loop()
-    yield loop
-    loop.close()
-
-
-# Test data fixtures
-@pytest.fixture
-def sample_search_request():
-    """Sample search request data."""
-    return {
-        "query": "Python web scraping",
-        "engines": ["google", "bing"],
-        "max_results": 10,
-        "scrape_content": True,
-        "language": "en",
-        "safe_search": "moderate"
-    }
-
-
-@pytest.fixture
-def sample_search_result():
-    """Sample search result data."""
-    return {
-        "rank": 1,
-        "title": "Python Web Scraping Tutorial",
-        "url": "https://example.com/tutorial",
-        "snippet": "Learn how to scrape websites with Python...",
-        "engine": "google"
-    }
-
-
-@pytest.fixture
-def sample_scraped_content():
-    """Sample scraped content data."""
-    return {
-        "url": "https://example.com/tutorial",
-        "title": "Python Web Scraping Tutorial",
-        "text": "This is a comprehensive guide to web scraping with Python...",
-        "images": ["https://example.com/img1.jpg"],
-        "links": ["https://example.com/related"],
-        "extraction_success": True,
-        "extraction_time_ms": 250,
-        "word_count": 1500,
-        "language_detected": "en",
-        "content_quality_score": 0.85,
-        "metadata": {
-            "title": "Python Web Scraping Tutorial",
-            "description": "Learn web scraping with Python",
-            "author": "John Doe",
-            "keywords": ["python", "web scraping", "tutorial"]
-        }
-    }
diff --git a/apps/backend/tests/e2e/test_complete_flows.py b/apps/backend/tests/e2e/test_complete_flows.py
deleted file mode 100644
index bfd87f2..0000000
--- a/apps/backend/tests/e2e/test_complete_flows.py
+++ /dev/null
@@ -1,620 +0,0 @@
-"""
-End-to-end tests for complete user flows in the  API.
-These tests run against a fully deployed environment.
-"""
-import asyncio
-import json
-import time
-from typing import Dict, List, Any
-import pytest
-import httpx
-from httpx import AsyncClient
-import os
-from datetime import datetime, timedelta
-
-# Configuration
-BASE_URL = os.getenv("E2E_BASE_URL", "http://localhost:8000")
-API_KEY = os.getenv("E2E_API_KEY", "test-api-key")
-WEBHOOK_URL = os.getenv("E2E_WEBHOOK_URL", "https://webhook.site/test")
-
-
-class TestUnSearchE2E:
-    """Complete end-to-end test scenarios."""
-    
-    @pytest.fixture
-    async def client(self):
-        """Create authenticated HTTP client."""
-        async with AsyncClient(
-            base_url=BASE_URL,
-            headers={"X-API-Key": API_KEY},
-            timeout=30.0
-        ) as client:
-            yield client
-    
-    @pytest.mark.asyncio
-    async def test_complete_search_and_scrape_flow(self, client: AsyncClient):
-        """
-        Test the complete flow of searching and scraping content.
-        
-        Flow:
-        1. Search for content
-        2. Verify search results
-        3. Check scraped content
-        4. Validate response format
-        """
-        # Step 1: Perform search with scraping
-        request_data = {
-            "query": "Python FastAPI tutorial",
-            "engines": ["google", "bing"],
-            "max_results": 5,
-            "scrape_content": True,
-            "include_images": True,
-            "include_links": True,
-            "cache_ttl": 3600,
-            "language": "en",
-            "safe_search": "moderate"
-        }
-        
-        start_time = time.time()
-        response = await client.post("/api/v1/search", json=request_data)
-        response_time = (time.time() - start_time) * 1000
-        
-        # Verify response status
-        assert response.status_code == 200, f"Expected 200, got {response.status_code}"
-        
-        # Verify response time is reasonable
-        assert response_time < 10000, f"Response took too long: {response_time}ms"
-        
-        data = response.json()
-        
-        # Step 2: Verify response structure
-        assert "search_metadata" in data
-        assert "results" in data
-        assert "processing_time_ms" in data
-        assert "cached" in data
-        assert "total_results" in data
-        
-        # Verify metadata
-        metadata = data["search_metadata"]
-        assert metadata["query"] == request_data["query"]
-        assert set(metadata["engines"]) == set(request_data["engines"])
-        assert metadata["language"] == request_data["language"]
-        
-        # Step 3: Verify search results
-        results = data["results"]
-        assert len(results) > 0, "No results returned"
-        assert len(results) <= request_data["max_results"]
-        
-        # Check each result
-        for result in results:
-            assert "rank" in result
-            assert "title" in result
-            assert "url" in result
-            assert "snippet" in result
-            assert "engine" in result
-            
-            # Verify scraped content if available
-            if "scraped_content" in result and result["scraped_content"]:
-                content = result["scraped_content"]
-                assert "text" in content
-                assert "metadata" in content
-                assert "extraction_success" in content
-                assert "word_count" in content
-                
-                # Check requested fields
-                if request_data["include_images"]:
-                    assert "images" in content
-                if request_data["include_links"]:
-                    assert "links" in content
-        
-        # Step 4: Test caching
-        # Make the same request again
-        cached_response = await client.post("/api/v1/search", json=request_data)
-        assert cached_response.status_code == 200
-        cached_data = cached_response.json()
-        
-        # Should be cached
-        assert cached_data.get("cached", False) == True
-        
-        # Results should be the same
-        assert len(cached_data["results"]) == len(data["results"])
-    
-    @pytest.mark.asyncio
-    async def test_batch_search_flow(self, client: AsyncClient):
-        """
-        Test batch search functionality.
-        
-        Flow:
-        1. Submit multiple searches
-        2. Verify batch processing
-        3. Check individual results
-        """
-        batch_request = {
-            "searches": [
-                {
-                    "query": "machine learning algorithms",
-                    "engines": ["google"],
-                    "max_results": 3
-                },
-                {
-                    "query": "deep learning frameworks",
-                    "engines": ["bing"],
-                    "max_results": 3
-                },
-                {
-                    "query": "neural networks tutorial",
-                    "engines": ["duckduckgo"],
-                    "max_results": 3
-                }
-            ]
-        }
-        
-        response = await client.post("/api/v1/search/batch", json=batch_request)
-        
-        # Check if batch endpoint exists
-        if response.status_code == 404:
-            pytest.skip("Batch endpoint not implemented")
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert "batch_id" in data
-        assert "results" in data
-        assert len(data["results"]) == len(batch_request["searches"])
-        
-        # Verify each search result
-        for i, result in enumerate(data["results"]):
-            assert result["query"] == batch_request["searches"][i]["query"]
-            assert "results" in result
-            assert len(result["results"]) <= batch_request["searches"][i]["max_results"]
-    
-    @pytest.mark.asyncio
-    async def test_async_processing_flow(self, client: AsyncClient):
-        """
-        Test asynchronous processing with webhooks.
-        
-        Flow:
-        1. Submit async search request
-        2. Get job ID
-        3. Check job status
-        4. Verify completion
-        """
-        async_request = {
-            "query": "large dataset processing techniques",
-            "engines": ["google", "bing", "duckduckgo"],
-            "max_results": 20,
-            "scrape_content": True,
-            "async_mode": True,
-            "webhook_url": WEBHOOK_URL
-        }
-        
-        # Submit async request
-        response = await client.post("/api/v1/search", json=async_request)
-        
-        # Check if async mode is implemented
-        if "job_id" not in response.json():
-            pytest.skip("Async mode not implemented")
-        
-        assert response.status_code == 202  # Accepted
-        data = response.json()
-        
-        assert "job_id" in data
-        assert "status" in data
-        assert data["status"] in ["pending", "processing"]
-        
-        job_id = data["job_id"]
-        
-        # Poll for job completion (max 60 seconds)
-        max_attempts = 30
-        for attempt in range(max_attempts):
-            status_response = await client.get(f"/api/v1/search/status/{job_id}")
-            
-            if status_response.status_code == 200:
-                status_data = status_response.json()
-                
-                if status_data["status"] == "completed":
-                    assert "results" in status_data
-                    assert len(status_data["results"]) > 0
-                    break
-                elif status_data["status"] == "failed":
-                    pytest.fail(f"Job failed: {status_data.get('error')}")
-            
-            await asyncio.sleep(2)
-        else:
-            pytest.fail(f"Job {job_id} did not complete in time")
-    
-    @pytest.mark.asyncio
-    async def test_error_handling_flow(self, client: AsyncClient):
-        """
-        Test error handling and recovery.
-        
-        Flow:
-        1. Send invalid requests
-        2. Verify error responses
-        3. Check rate limiting
-        4. Test recovery
-        """
-        # Test 1: Invalid query
-        invalid_request = {
-            "query": "",  # Empty query
-            "engines": ["google"]
-        }
-        
-        response = await client.post("/api/v1/search", json=invalid_request)
-        assert response.status_code == 422  # Validation error
-        error_data = response.json()
-        assert "error" in error_data or "detail" in error_data
-        
-        # Test 2: Invalid engine
-        invalid_engine_request = {
-            "query": "test",
-            "engines": ["invalid_engine"]
-        }
-        
-        response = await client.post("/api/v1/search", json=invalid_engine_request)
-        assert response.status_code in [400, 422]
-        
-        # Test 3: Exceed max results
-        excessive_request = {
-            "query": "test",
-            "engines": ["google"],
-            "max_results": 1000  # Exceeds limit
-        }
-        
-        response = await client.post("/api/v1/search", json=excessive_request)
-        assert response.status_code in [400, 422]
-        
-        # Test 4: Rate limiting (if enabled)
-        # Make many requests quickly
-        rate_limit_hit = False
-        for i in range(100):
-            response = await client.post("/api/v1/search", json={
-                "query": f"rate limit test {i}",
-                "engines": ["google"],
-                "max_results": 1
-            })
-            
-            if response.status_code == 429:  # Too Many Requests
-                rate_limit_hit = True
-                break
-        
-        # Note: Rate limiting might not be hit in test environment
-        if rate_limit_hit:
-            assert "Retry-After" in response.headers
-    
-    @pytest.mark.asyncio
-    async def test_multilanguage_search_flow(self, client: AsyncClient):
-        """
-        Test searching in different languages.
-        
-        Flow:
-        1. Search in multiple languages
-        2. Verify language-specific results
-        3. Check content language detection
-        """
-        languages = [
-            ("en", "Python programming"),
-            ("es", "programación Python"),
-            ("fr", "programmation Python"),
-            ("de", "Python Programmierung")
-        ]
-        
-        for lang_code, query in languages:
-            request_data = {
-                "query": query,
-                "engines": ["google"],
-                "max_results": 3,
-                "language": lang_code,
-                "scrape_content": True
-            }
-            
-            response = await client.post("/api/v1/search", json=request_data)
-            assert response.status_code == 200
-            
-            data = response.json()
-            assert data["search_metadata"]["language"] == lang_code
-            
-            # Check if results contain content in the expected language
-            results = data["results"]
-            if results and results[0].get("scraped_content"):
-                content = results[0]["scraped_content"]
-                if "language_detected" in content:
-                    # Language detection might not be 100% accurate
-                    # but should be close
-                    detected_lang = content["language_detected"]
-                    print(f"Query language: {lang_code}, Detected: {detected_lang}")
-    
-    @pytest.mark.asyncio
-    async def test_custom_selectors_flow(self, client: AsyncClient):
-        """
-        Test custom CSS selector functionality.
-        
-        Flow:
-        1. Search with custom selectors
-        2. Verify extracted content
-        3. Test fallback behavior
-        """
-        request_data = {
-            "query": "example.com",
-            "engines": ["google"],
-            "max_results": 1,
-            "scrape_content": True,
-            "scrape_selectors": {
-                "title": "h1, h2, title",
-                "main_content": "main, article, .content",
-                "navigation": "nav, .navigation",
-                "footer": "footer, .footer"
-            }
-        }
-        
-        response = await client.post("/api/v1/search", json=request_data)
-        
-        if response.status_code == 200:
-            data = response.json()
-            results = data["results"]
-            
-            if results and results[0].get("scraped_content"):
-                content = results[0]["scraped_content"]
-                
-                # Check if custom extraction worked
-                if "custom_extracted" in content:
-                    extracted = content["custom_extracted"]
-                    assert isinstance(extracted, dict)
-                    
-                    # Verify requested selectors were attempted
-                    for selector_name in request_data["scrape_selectors"]:
-                        # The content might not exist, but the key should be present
-                        assert selector_name in extracted or True  # Flexible check
-    
-    @pytest.mark.asyncio
-    async def test_performance_and_reliability(self, client: AsyncClient):
-        """
-        Test system performance and reliability under load.
-        
-        Flow:
-        1. Send concurrent requests
-        2. Measure response times
-        3. Verify consistency
-        4. Check error rates
-        """
-        # Define test parameters
-        num_concurrent = 10
-        num_iterations = 3
-        
-        async def make_search_request(query_suffix: int):
-            """Make a single search request."""
-            request_data = {
-                "query": f"performance test query {query_suffix}",
-                "engines": ["google"],
-                "max_results": 2,
-                "scrape_content": False  # Faster without scraping
-            }
-            
-            start = time.time()
-            try:
-                response = await client.post("/api/v1/search", json=request_data)
-                duration = time.time() - start
-                return {
-                    "success": response.status_code == 200,
-                    "duration": duration,
-                    "status_code": response.status_code
-                }
-            except Exception as e:
-                return {
-                    "success": False,
-                    "duration": time.time() - start,
-                    "error": str(e)
-                }
-        
-        # Run performance test
-        all_results = []
-        for iteration in range(num_iterations):
-            # Create concurrent tasks
-            tasks = [
-                make_search_request(i + (iteration * num_concurrent))
-                for i in range(num_concurrent)
-            ]
-            
-            # Execute concurrently
-            results = await asyncio.gather(*tasks)
-            all_results.extend(results)
-            
-            # Small delay between iterations
-            await asyncio.sleep(1)
-        
-        # Analyze results
-        successful_requests = [r for r in all_results if r["success"]]
-        failed_requests = [r for r in all_results if not r["success"]]
-        
-        success_rate = len(successful_requests) / len(all_results)
-        assert success_rate >= 0.95, f"Success rate too low: {success_rate:.2%}"
-        
-        # Calculate response time statistics
-        if successful_requests:
-            response_times = [r["duration"] for r in successful_requests]
-            avg_time = sum(response_times) / len(response_times)
-            max_time = max(response_times)
-            min_time = min(response_times)
-            
-            print(f"\nPerformance Statistics:")
-            print(f"  Success Rate: {success_rate:.2%}")
-            print(f"  Avg Response Time: {avg_time:.2f}s")
-            print(f"  Min Response Time: {min_time:.2f}s")
-            print(f"  Max Response Time: {max_time:.2f}s")
-            
-            # Performance assertions
-            assert avg_time < 5.0, f"Average response time too high: {avg_time:.2f}s"
-            assert max_time < 10.0, f"Max response time too high: {max_time:.2f}s"
-    
-    @pytest.mark.asyncio
-    async def test_api_versioning_and_compatibility(self, client: AsyncClient):
-        """
-        Test API versioning and backward compatibility.
-        
-        Flow:
-        1. Test current version endpoint
-        2. Test deprecated features (if any)
-        3. Verify version headers
-        """
-        # Test version endpoint
-        response = await client.get("/api/v1/version")
-        if response.status_code == 200:
-            version_data = response.json()
-            assert "version" in version_data
-            assert "api_version" in version_data
-        
-        # Test that v1 endpoints work
-        v1_response = await client.post("/api/v1/search", json={
-            "query": "test",
-            "engines": ["google"],
-            "max_results": 1
-        })
-        assert v1_response.status_code == 200
-        
-        # Check for API version headers
-        assert "X-API-Version" in v1_response.headers or True  # Flexible check
-
-
-class TestHealthAndMonitoring:
-    """E2E tests for health checks and monitoring endpoints."""
-    
-    @pytest.fixture
-    async def client(self):
-        """Create HTTP client without authentication for public endpoints."""
-        async with AsyncClient(base_url=BASE_URL, timeout=10.0) as client:
-            yield client
-    
-    @pytest.mark.asyncio
-    async def test_health_endpoint(self, client: AsyncClient):
-        """Test the health check endpoint."""
-        response = await client.get("/health")
-        assert response.status_code == 200
-        
-        data = response.json()
-        assert "status" in data
-        assert data["status"] in ["healthy", "degraded"]
-        
-        # Check service statuses
-        if "services" in data:
-            services = data["services"]
-            expected_services = ["api", "database", "redis", "searxng"]
-            
-            for service in expected_services:
-                if service in services:
-                    assert "status" in services[service]
-                    assert "response_time_ms" in services[service]
-    
-    @pytest.mark.asyncio
-    async def test_metrics_endpoint(self, client: AsyncClient):
-        """Test the Prometheus metrics endpoint."""
-        response = await client.get("/metrics")
-        
-        if response.status_code == 200:
-            content = response.text
-            
-            # Check for standard Prometheus metrics
-            assert "http_requests_total" in content or "request_count" in content
-            assert "http_request_duration_seconds" in content or "request_duration" in content
-            
-            # Check for custom metrics
-            expected_metrics = [
-                "search_requests_total",
-                "search_results_count",
-                "scraping_success_rate",
-                "cache_hit_ratio"
-            ]
-            
-            for metric in expected_metrics:
-                # Metrics might not be present if not used yet
-                pass  # Flexible check
-    
-    @pytest.mark.asyncio
-    async def test_documentation_endpoints(self, client: AsyncClient):
-        """Test API documentation endpoints."""
-        # Test OpenAPI schema
-        openapi_response = await client.get("/openapi.json")
-        if openapi_response.status_code == 200:
-            schema = openapi_response.json()
-            assert "openapi" in schema
-            assert "paths" in schema
-            assert "/api/v1/search" in schema["paths"]
-        
-        # Test Swagger UI
-        docs_response = await client.get("/docs")
-        if docs_response.status_code == 200:
-            assert "swagger" in docs_response.text.lower() or "openapi" in docs_response.text.lower()
-        
-        # Test ReDoc (if available)
-        redoc_response = await client.get("/redoc")
-        # ReDoc might not be configured, so we don't assert on status
-
-
-@pytest.mark.asyncio
-class TestDataIntegrity:
-    """E2E tests for data integrity and consistency."""
-    
-    @pytest.fixture
-    async def client(self):
-        """Create authenticated HTTP client."""
-        async with AsyncClient(
-            base_url=BASE_URL,
-            headers={"X-API-Key": API_KEY},
-            timeout=30.0
-        ) as client:
-            yield client
-    
-    async def test_data_consistency_across_requests(self, client: AsyncClient):
-        """
-        Test that data remains consistent across multiple requests.
-        
-        Flow:
-        1. Make identical requests
-        2. Verify consistency (when not cached)
-        3. Test with different parameters
-        """
-        # Disable caching for this test
-        request_data = {
-            "query": "data consistency test " + str(datetime.now()),
-            "engines": ["google"],
-            "max_results": 5,
-            "cache_ttl": 0  # Disable caching
-        }
-        
-        # Make multiple identical requests
-        responses = []
-        for _ in range(3):
-            response = await client.post("/api/v1/search", json=request_data)
-            assert response.status_code == 200
-            responses.append(response.json())
-            await asyncio.sleep(1)  # Small delay between requests
-        
-        # Verify that the structure is consistent
-        for i in range(1, len(responses)):
-            assert len(responses[i]["results"]) == len(responses[0]["results"])
-            assert responses[i]["search_metadata"]["query"] == responses[0]["search_metadata"]["query"]
-    
-    async def test_unicode_and_special_characters(self, client: AsyncClient):
-        """Test handling of Unicode and special characters."""
-        test_queries = [
-            "Python 编程",  # Chinese
-            "Программирование на Python",  # Russian
-            "Python & C++ comparison",
-            "Search with \"quotes\" and 'apostrophes'",
-            "Special chars: !@#$%^&*()",
-            "Emoji test 🐍 🔍 💻"
-        ]
-        
-        for query in test_queries:
-            response = await client.post("/api/v1/search", json={
-                "query": query,
-                "engines": ["google"],
-                "max_results": 1
-            })
-            
-            # Should handle all characters gracefully
-            assert response.status_code in [200, 400, 422]
-            
-            if response.status_code == 200:
-                data = response.json()
-                # Query should be preserved correctly
-                assert data["search_metadata"]["query"] == query
diff --git a/apps/backend/tests/integration/test_api.py b/apps/backend/tests/integration/test_api.py
deleted file mode 100644
index 973affe..0000000
--- a/apps/backend/tests/integration/test_api.py
+++ /dev/null
@@ -1,244 +0,0 @@
-"""
-Integration tests for API endpoints.
-"""
-import pytest
-from httpx import AsyncClient
-from unittest.mock import AsyncMock, MagicMock
-
-from app.models.responses import SearchResult, EngineInfo
-
-
-@pytest.mark.asyncio
-class TestSearchEndpoints:
-    """Test search API endpoints."""
-    
-    async def test_search_endpoint_success(
-        self,
-        authenticated_client: AsyncClient,
-        sample_search_request,
-        mock_searxng,
-        mock_scraper,
-        test_cache,
-        test_db
-    ):
-        """Test successful search and scrape operation."""
-        # Mock search results
-        mock_searxng.search = AsyncMock(return_value=[
-            SearchResult(
-                rank=1,
-                title="Test Result",
-                url="https://example.com",
-                snippet="Test snippet",
-                engine="google"
-            )
-        ])
-        
-        # Mock dependencies
-        authenticated_client.app.dependency_overrides[get_searxng_service] = lambda: mock_searxng
-        authenticated_client.app.dependency_overrides[get_scraping_service] = lambda: mock_scraper
-        authenticated_client.app.dependency_overrides[get_cache_service] = lambda: test_cache
-        authenticated_client.app.dependency_overrides[get_database_service] = lambda: test_db
-        
-        response = await authenticated_client.post(
-            "/api/v1/search",
-            json=sample_search_request
-        )
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert "search_metadata" in data
-        assert "results" in data
-        assert len(data["results"]) == 1
-        assert data["results"][0]["title"] == "Test Result"
-        assert data["cached"] is False
-        
-    async def test_search_endpoint_unauthorized(
-        self,
-        client: AsyncClient,
-        sample_search_request
-    ):
-        """Test search without authentication."""
-        response = await client.post(
-            "/api/v1/search",
-            json=sample_search_request
-        )
-        
-        assert response.status_code == 401
-        assert "API key required" in response.json()["detail"]
-        
-    async def test_search_endpoint_invalid_request(
-        self,
-        authenticated_client: AsyncClient
-    ):
-        """Test search with invalid request data."""
-        response = await authenticated_client.post(
-            "/api/v1/search",
-            json={
-                "query": "",  # Empty query
-                "engines": ["invalid_engine"]
-            }
-        )
-        
-        assert response.status_code == 422
-        
-    async def test_search_endpoint_with_caching(
-        self,
-        authenticated_client: AsyncClient,
-        sample_search_request,
-        mock_searxng,
-        test_cache
-    ):
-        """Test search with caching enabled."""
-        # First request - cache miss
-        mock_searxng.search = AsyncMock(return_value=[
-            SearchResult(
-                rank=1,
-                title="Cached Result",
-                url="https://example.com",
-                snippet="Test",
-                engine="google"
-            )
-        ])
-        
-        authenticated_client.app.dependency_overrides[get_searxng_service] = lambda: mock_searxng
-        authenticated_client.app.dependency_overrides[get_cache_service] = lambda: test_cache
-        
-        response1 = await authenticated_client.post(
-            "/api/v1/search",
-            json=sample_search_request
-        )
-        
-        assert response1.status_code == 200
-        data1 = response1.json()
-        assert data1["cached"] is False
-        
-        # Second request - should hit cache
-        response2 = await authenticated_client.post(
-            "/api/v1/search",
-            json=sample_search_request
-        )
-        
-        assert response2.status_code == 200
-        data2 = response2.json()
-        # Note: In real implementation, this would be True after proper cache implementation
-        
-    async def test_batch_search_endpoint(
-        self,
-        authenticated_client: AsyncClient,
-        mock_searxng
-    ):
-        """Test batch search endpoint."""
-        mock_searxng.search = AsyncMock(side_effect=[
-            [SearchResult(rank=1, title=f"Result for query {i}", 
-                         url=f"https://example{i}.com", snippet="Test", engine="google")]
-            for i in range(3)
-        ])
-        
-        authenticated_client.app.dependency_overrides[get_searxng_service] = lambda: mock_searxng
-        
-        response = await authenticated_client.post(
-            "/api/v1/search/batch",
-            json={
-                "queries": ["Python", "FastAPI", "Docker"],
-                "engines": ["google"],
-                "max_results_per_query": 5
-            }
-        )
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert data["queries_processed"] == 3
-        assert data["queries_failed"] == 0
-        assert len(data["results"]) == 3
-        
-    async def test_list_engines_endpoint(
-        self,
-        authenticated_client: AsyncClient,
-        mock_searxng
-    ):
-        """Test list engines endpoint."""
-        mock_engines = {
-            "google": EngineInfo(
-                name="google",
-                enabled=True,
-                categories=["general"],
-                supported_languages=["*"],
-                safe_search_support=True,
-                time_range_support=True,
-                paging_support=True
-            ),
-            "bing": EngineInfo(
-                name="bing",
-                enabled=True,
-                categories=["general"],
-                supported_languages=["*"],
-                safe_search_support=True,
-                time_range_support=True,
-                paging_support=True
-            )
-        }
-        
-        mock_searxng.get_available_engines = AsyncMock(return_value=mock_engines)
-        authenticated_client.app.dependency_overrides[get_searxng_service] = lambda: mock_searxng
-        
-        response = await authenticated_client.get("/api/v1/search/engines")
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert data["total_engines"] == 2
-        assert data["enabled_engines"] == 2
-        assert "google" in data["engines"]
-        assert "bing" in data["engines"]
-
-
-@pytest.mark.asyncio
-class TestHealthEndpoints:
-    """Test health check endpoints."""
-    
-    async def test_basic_health_check(self, client: AsyncClient):
-        """Test basic health endpoint."""
-        response = await client.get("/health")
-        
-        assert response.status_code == 200
-        assert response.json()["status"] == "healthy"
-        
-    async def test_detailed_health_check(
-        self,
-        client: AsyncClient,
-        mock_searxng,
-        test_cache,
-        test_db
-    ):
-        """Test detailed health check endpoint."""
-        from app.models.responses import ServiceHealth
-        
-        mock_searxng.health_check = AsyncMock(return_value=ServiceHealth(
-            status="healthy",
-            latency_ms=50,
-            last_check="2024-01-01T00:00:00"
-        ))
-        
-        client.app.dependency_overrides[get_searxng_service] = lambda: mock_searxng
-        client.app.dependency_overrides[get_cache_service] = lambda: test_cache
-        client.app.dependency_overrides[get_database_service] = lambda: test_db
-        
-        response = await client.get("/api/v1/search/health")
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert data["status"] in ["healthy", "degraded", "unhealthy"]
-        assert "services" in data
-        assert "searxng" in data["services"]
-        assert "redis" in data["services"]
-        assert "database" in data["services"]
-
-
-# Import after to avoid circular imports
-from app.services.searxng import get_searxng_service
-from app.services.scraping import get_scraping_service
-from app.services.cache import get_cache_service
-from app.services.database import get_database_service
diff --git a/apps/backend/tests/integration/test_endpoints.py b/apps/backend/tests/integration/test_endpoints.py
deleted file mode 100644
index 125371a..0000000
--- a/apps/backend/tests/integration/test_endpoints.py
+++ /dev/null
@@ -1,398 +0,0 @@
-"""
-Integration tests for API endpoints.
-"""
-import pytest
-import asyncio
-from unittest.mock import Mock, AsyncMock, patch
-from fastapi.testclient import TestClient
-import httpx
-
-from app.main import app
-from app.models.requests import UnSearchRequest
-from app.models.responses import SearchResult, SearchMetadata
-from app.config import get_settings
-
-
-@pytest.fixture
-def client():
-    """Create test client."""
-    return TestClient(app)
-
-
-@pytest.fixture
-def mock_services():
-    """Mock all external services."""
-    with patch('app.services.searxng.get_searxng_service') as mock_searxng, \
-         patch('app.services.scraping.get_scraping_service') as mock_scraper, \
-         patch('app.services.cache.get_cache_service') as mock_cache, \
-         patch('app.services.database.get_database_service') as mock_db:
-        
-        # Mock SearXNG service
-        searxng_mock = AsyncMock()
-        searxng_mock.search = AsyncMock(return_value=[
-            SearchResult(
-                rank=1,
-                title="Test Result",
-                url="https://example.com",
-                snippet="Test snippet",
-                engine="google"
-            )
-        ])
-        searxng_mock.health_check = AsyncMock(return_value=Mock(status="healthy", latency_ms=100))
-        searxng_mock.get_available_engines = AsyncMock(return_value={})
-        mock_searxng.return_value = searxng_mock
-        
-        # Mock scraping service
-        scraper_mock = AsyncMock()
-        scraper_mock.scrape_urls = AsyncMock(return_value=[])
-        mock_scraper.return_value = scraper_mock
-        
-        # Mock cache service
-        cache_mock = AsyncMock()
-        cache_mock.get_search_results = AsyncMock(return_value=None)
-        cache_mock.set_search_results = AsyncMock()
-        cache_mock.generate_cache_key = Mock(return_value="test-cache-key")
-        cache_mock._client = AsyncMock()
-        cache_mock._client.ping = AsyncMock()
-        mock_cache.return_value = cache_mock
-        
-        # Mock database service
-        db_mock = AsyncMock()
-        db_mock.get_api_key = AsyncMock(return_value=None)
-        db_mock.log_search_request = AsyncMock()
-        db_mock.log_error = AsyncMock()
-        db_mock.get_session = AsyncMock()
-        db_mock.get_session.return_value.__aenter__ = AsyncMock()
-        db_mock.get_session.return_value.__aexit__ = AsyncMock()
-        mock_db.return_value = db_mock
-        
-        yield {
-            'searxng': searxng_mock,
-            'scraper': scraper_mock,
-            'cache': cache_mock,
-            'db': db_mock
-        }
-
-
-class TestSearchEndpoints:
-    """Test search-related endpoints."""
-    
-    def test_search_scrape_basic(self, client, mock_services):
-        """Test basic search and scrape."""
-        # Disable API key requirement for testing
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []  # No API keys required
-            mock_settings.return_value = settings
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "python programming",
-                "engines": ["google"],
-                "max_results": 5,
-                "scrape_content": False
-            })
-            
-            assert response.status_code == 200
-            data = response.json()
-            
-            assert "search_metadata" in data
-            assert "results" in data
-            assert "processing_time_ms" in data
-            assert data["search_metadata"]["query"] == "python programming"
-    
-    def test_search_scrape_with_content(self, client, mock_services):
-        """Test search with content scraping."""
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "python programming",
-                "engines": ["google"],
-                "max_results": 5,
-                "scrape_content": True,
-                "include_images": True,
-                "include_links": True
-            })
-            
-            assert response.status_code == 200
-            data = response.json()
-            
-            assert data["search_metadata"]["query"] == "python programming"
-            mock_services['scraper'].scrape_urls.assert_called_once()
-    
-    def test_search_validation_errors(self, client, mock_services):
-        """Test request validation errors."""
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            # Empty query
-            response = client.post("/api/v1/search/", json={
-                "query": "",
-                "engines": ["google"]
-            })
-            assert response.status_code == 422
-            
-            # Invalid engine
-            response = client.post("/api/v1/search/", json={
-                "query": "test",
-                "engines": ["invalid_engine"]
-            })
-            assert response.status_code == 422
-            
-            # Too many results
-            response = client.post("/api/v1/search/", json={
-                "query": "test",
-                "engines": ["google"],
-                "max_results": 200
-            })
-            assert response.status_code == 422
-    
-    def test_search_with_api_key(self, client, mock_services):
-        """Test search with API key authentication."""
-        from app.models.database import APIKey
-        
-        # Mock API key in database
-        api_key_obj = APIKey(id=1, key="test-api-key", name="Test Key", is_active=True)
-        mock_services['db'].get_api_key.return_value = api_key_obj
-        
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = ["test-api-key"]  # Require API key
-            mock_settings.return_value = settings
-            
-            # Request with valid API key
-            response = client.post("/api/v1/search/", 
-                headers={"X-API-Key": "test-api-key"},
-                json={
-                    "query": "python programming",
-                    "engines": ["google"]
-                }
-            )
-            
-            assert response.status_code == 200
-    
-    def test_search_unauthorized(self, client, mock_services):
-        """Test unauthorized access."""
-        mock_services['db'].get_api_key.return_value = None
-        
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = ["required-key"]  # Require API key
-            mock_settings.return_value = settings
-            
-            # Request without API key
-            response = client.post("/api/v1/search/", json={
-                "query": "python programming",
-                "engines": ["google"]
-            })
-            
-            assert response.status_code == 401
-            
-            # Request with invalid API key
-            response = client.post("/api/v1/search/", 
-                headers={"X-API-Key": "invalid-key"},
-                json={
-                    "query": "python programming",
-                    "engines": ["google"]
-                }
-            )
-            
-            assert response.status_code == 401
-    
-    def test_batch_search(self, client, mock_services):
-        """Test batch search endpoint."""
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            response = client.post("/api/v1/search/batch", json={
-                "queries": ["python programming", "web scraping"],
-                "engines": ["google"],
-                "max_results_per_query": 3,
-                "parallel_requests": 2
-            })
-            
-            assert response.status_code == 200
-            data = response.json()
-            
-            assert "batch_id" in data
-            assert "queries_processed" in data
-            assert "results" in data
-            assert data["queries_processed"] >= 0
-    
-    def test_list_engines(self, client, mock_services):
-        """Test engines listing endpoint."""
-        mock_engines = {
-            "google": Mock(name="google", enabled=True),
-            "bing": Mock(name="bing", enabled=True)
-        }
-        mock_services['searxng'].get_available_engines.return_value = mock_engines
-        
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            response = client.get("/api/v1/search/engines")
-            
-            assert response.status_code == 200
-            data = response.json()
-            
-            assert "engines" in data
-            assert "total_engines" in data
-            assert "enabled_engines" in data
-    
-    def test_health_check(self, client, mock_services):
-        """Test health check endpoint."""
-        response = client.get("/api/v1/search/health")
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        assert "status" in data
-        assert "version" in data
-        assert "services" in data
-        assert "timestamp" in data
-
-
-class TestErrorHandling:
-    """Test error handling."""
-    
-    def test_searxng_service_error(self, client, mock_services):
-        """Test SearXNG service error handling."""
-        # Mock SearXNG service error
-        mock_services['searxng'].search.side_effect = Exception("SearXNG connection failed")
-        
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "test query",
-                "engines": ["google"]
-            })
-            
-            assert response.status_code == 500
-            data = response.json()
-            assert "error" in data
-    
-    def test_rate_limiting(self, client, mock_services):
-        """Test rate limiting."""
-        # This would require setting up actual rate limiting
-        # For now, just test that the endpoint accepts requests
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            settings.rate_limit_enabled = True
-            mock_settings.return_value = settings
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "test query",
-                "engines": ["google"]
-            })
-            
-            # Should still work for single request
-            assert response.status_code in [200, 429]  # Either success or rate limited
-
-
-class TestAsyncOperations:
-    """Test async operations."""
-    
-    def test_async_search_request(self, client, mock_services):
-        """Test async search request creation."""
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            # Mock job creation
-            from app.models.database import ScrapingJob
-            mock_job = ScrapingJob(job_id="test-job-123")
-            mock_services['db'].create_scraping_job.return_value = mock_job
-            mock_services['db'].update_scraping_job.return_value = mock_job
-            
-            with patch('app.workers.tasks.process_async_search_scrape.delay') as mock_task:
-                mock_task.return_value = Mock(id="task-123")
-                
-                response = client.post("/api/v1/search/", json={
-                    "query": "test query",
-                    "engines": ["google"],
-                    "async_mode": True,
-                    "webhook_url": "https://example.com/webhook"
-                })
-                
-                assert response.status_code == 200
-                data = response.json()
-                
-                assert "task_id" in data
-                assert "status" in data
-                assert data["status"] in ["pending", "processing"]
-
-
-class TestCaching:
-    """Test caching functionality."""
-    
-    def test_cache_hit(self, client, mock_services):
-        """Test cache hit scenario."""
-        # Mock cached response
-        from app.models.responses import UnSearchResponse, SearchMetadata
-        
-        cached_response = UnSearchResponse(
-            search_metadata=SearchMetadata(
-                query="cached query",
-                engines_used=["google"],
-                engines_succeeded=["google"],
-                engines_failed=[],
-                total_results_found=1,
-                results_returned=1,
-                search_time_ms=100
-            ),
-            results=[],
-            processing_time_ms=50,
-            cached=True,
-            total_results=1,
-            request_id="cached-123"
-        )
-        
-        mock_services['cache'].get_search_results.return_value = cached_response
-        
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "cached query",
-                "engines": ["google"],
-                "cache_ttl": 3600
-            })
-            
-            assert response.status_code == 200
-            data = response.json()
-            
-            assert data["cached"] is True
-            # SearXNG should not be called for cached results
-            mock_services['searxng'].search.assert_not_called()
-    
-    def test_cache_disabled(self, client, mock_services):
-        """Test when caching is disabled."""
-        with patch('app.config.get_settings') as mock_settings:
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "test query",
-                "engines": ["google"],
-                "cache_ttl": 0  # Disable caching
-            })
-            
-            assert response.status_code == 200
-            # Cache should not be checked when TTL is 0
-            mock_services['cache'].get_search_results.assert_not_called()
diff --git a/apps/backend/tests/performance/locustfile.py b/apps/backend/tests/performance/locustfile.py
deleted file mode 100644
index bf5393b..0000000
--- a/apps/backend/tests/performance/locustfile.py
+++ /dev/null
@@ -1,426 +0,0 @@
-"""
-Load testing configuration using Locust.
-Run with: locust -f locustfile.py --host http://localhost:8000
-"""
-import json
-import random
-import time
-from locust import HttpUser, task, between, events
-from locust.env import Environment
-from locust.stats import StatsCSVFileWriter
-import os
-
-
-# Configuration
-API_KEY = os.getenv("LOAD_TEST_API_KEY", "test-api-key")
-SEARCH_QUERIES = [
-    "Python programming",
-    "machine learning algorithms",
-    "web scraping techniques",
-    "FastAPI tutorial",
-    "data science tools",
-    "artificial intelligence",
-    "cloud computing AWS",
-    "Docker containers",
-    "Kubernetes orchestration",
-    "microservices architecture",
-    "REST API design",
-    "GraphQL vs REST",
-    "database optimization",
-    "Redis caching strategies",
-    "PostgreSQL performance",
-]
-
-SEARCH_ENGINES = [
-    ["google"],
-    ["bing"],
-    ["duckduckgo"],
-    ["google", "bing"],
-    ["google", "bing", "duckduckgo"],
-]
-
-
-class UnSearchUser(HttpUser):
-    """
-    Simulates a user interacting with the UnSearch API.
-    """
-    wait_time = between(1, 5)  # Wait 1-5 seconds between requests
-    
-    def on_start(self):
-        """Called when a user starts."""
-        self.client.headers.update({
-            "X-API-Key": API_KEY,
-            "Content-Type": "application/json"
-        })
-        
-        # Test authentication
-        response = self.client.get("/health", name="Health Check")
-        if response.status_code != 200:
-            print(f"Warning: Health check returned {response.status_code}")
-    
-    @task(10)
-    def search_without_scraping(self):
-        """Perform a search without content scraping (most common)."""
-        query = random.choice(SEARCH_QUERIES)
-        engines = random.choice(SEARCH_ENGINES)
-        max_results = random.randint(5, 20)
-        
-        payload = {
-            "query": query,
-            "engines": engines,
-            "max_results": max_results,
-            "scrape_content": False,
-            "cache_ttl": 3600
-        }
-        
-        with self.client.post(
-            "/api/v1/search",
-            json=payload,
-            name="Search (No Scraping)",
-            catch_response=True
-        ) as response:
-            if response.status_code == 200:
-                data = response.json()
-                if "results" not in data:
-                    response.failure("No results in response")
-                elif len(data["results"]) == 0:
-                    response.failure("Empty results")
-            elif response.status_code == 429:
-                # Rate limited - this is expected under load
-                response.success()
-            else:
-                response.failure(f"Got status code {response.status_code}")
-    
-    @task(3)
-    def search_with_scraping(self):
-        """Perform a search with content scraping (resource intensive)."""
-        query = random.choice(SEARCH_QUERIES)
-        engines = ["google"]  # Single engine for scraping tests
-        max_results = random.randint(3, 5)  # Fewer results for scraping
-        
-        payload = {
-            "query": query,
-            "engines": engines,
-            "max_results": max_results,
-            "scrape_content": True,
-            "include_images": True,
-            "include_links": True,
-            "cache_ttl": 3600
-        }
-        
-        with self.client.post(
-            "/api/v1/search",
-            json=payload,
-            name="Search (With Scraping)",
-            catch_response=True,
-            timeout=30  # Longer timeout for scraping
-        ) as response:
-            if response.status_code == 200:
-                data = response.json()
-                if "results" in data:
-                    # Check if scraping worked
-                    scraped_count = sum(1 for r in data["results"] if r.get("scraped_content"))
-                    if scraped_count == 0:
-                        response.failure("No content was scraped")
-            elif response.status_code == 429:
-                response.success()
-            else:
-                response.failure(f"Got status code {response.status_code}")
-    
-    @task(2)
-    def search_with_caching(self):
-        """Test caching by repeating the same search."""
-        # Use a limited set of queries for cache testing
-        cache_queries = SEARCH_QUERIES[:5]
-        query = random.choice(cache_queries)
-        
-        payload = {
-            "query": query,
-            "engines": ["google"],
-            "max_results": 10,
-            "scrape_content": False,
-            "cache_ttl": 3600
-        }
-        
-        # First request
-        with self.client.post(
-            "/api/v1/search",
-            json=payload,
-            name="Search (Cache Test)",
-            catch_response=True
-        ) as response:
-            if response.status_code == 200:
-                data = response.json()
-                
-                # Second request (should be cached)
-                time.sleep(0.5)
-                with self.client.post(
-                    "/api/v1/search",
-                    json=payload,
-                    name="Search (Cached)",
-                    catch_response=True
-                ) as cached_response:
-                    if cached_response.status_code == 200:
-                        cached_data = cached_response.json()
-                        if cached_data.get("cached", False):
-                            # Successfully hit cache
-                            pass
-                        else:
-                            # Not cached - might be OK in high load
-                            pass
-    
-    @task(1)
-    def check_health(self):
-        """Periodically check API health."""
-        with self.client.get(
-            "/health",
-            name="Health Check",
-            catch_response=True
-        ) as response:
-            if response.status_code == 200:
-                data = response.json()
-                if data.get("status") not in ["healthy", "degraded"]:
-                    response.failure("Unexpected health status")
-    
-    @task(1)
-    def check_metrics(self):
-        """Check metrics endpoint."""
-        with self.client.get(
-            "/metrics",
-            name="Metrics",
-            catch_response=True
-        ) as response:
-            # Metrics might be protected or disabled
-            if response.status_code in [200, 401, 404]:
-                response.success()
-            else:
-                response.failure(f"Unexpected status: {response.status_code}")
-
-
-class AdminUser(HttpUser):
-    """
-    Simulates an admin user checking system status.
-    """
-    wait_time = between(10, 30)  # Less frequent checks
-    weight = 1  # Fewer admin users
-    
-    def on_start(self):
-        """Set up admin headers."""
-        self.client.headers.update({
-            "X-API-Key": API_KEY,
-            "Content-Type": "application/json"
-        })
-    
-    @task
-    def check_system_health(self):
-        """Check overall system health."""
-        endpoints = ["/health", "/metrics", "/docs"]
-        
-        for endpoint in endpoints:
-            with self.client.get(
-                endpoint,
-                name=f"Admin: {endpoint}",
-                catch_response=True
-            ) as response:
-                if response.status_code in [200, 307, 401, 404]:
-                    response.success()
-
-
-class MobileUser(HttpUser):
-    """
-    Simulates mobile app users with different patterns.
-    """
-    wait_time = between(2, 8)
-    weight = 3  # Mobile users are common
-    
-    def on_start(self):
-        """Mobile client setup."""
-        self.client.headers.update({
-            "X-API-Key": API_KEY,
-            "Content-Type": "application/json",
-            "User-Agent": "UnSearch-Mobile/1.0"
-        })
-    
-    @task(10)
-    def quick_search(self):
-        """Quick searches typical of mobile users."""
-        query = random.choice(SEARCH_QUERIES)
-        
-        payload = {
-            "query": query,
-            "engines": ["google"],  # Mobile users might use single engine
-            "max_results": 5,  # Fewer results for mobile
-            "scrape_content": False,
-            "cache_ttl": 7200  # Longer cache for mobile
-        }
-        
-        with self.client.post(
-            "/api/v1/search",
-            json=payload,
-            name="Mobile: Quick Search",
-            catch_response=True,
-            timeout=10
-        ) as response:
-            if response.status_code == 200:
-                data = response.json()
-                # Mobile expects fast responses
-                if response.elapsed.total_seconds() > 3:
-                    response.failure("Response too slow for mobile")
-            elif response.status_code == 429:
-                response.success()
-
-
-# Custom event handlers for detailed reporting
-@events.test_start.add_listener
-def on_test_start(environment, **kwargs):
-    """Called when test starts."""
-    print(f"Load test starting...")
-    print(f"Target host: {environment.host}")
-    print(f"Users: {environment.parsed_options.num_users}")
-    print(f"Spawn rate: {environment.parsed_options.spawn_rate}")
-
-
-@events.test_stop.add_listener
-def on_test_stop(environment, **kwargs):
-    """Called when test stops."""
-    print("\nLoad test completed!")
-    print("\nFinal Statistics:")
-    print(f"Total requests: {environment.stats.total.num_requests}")
-    print(f"Failure rate: {environment.stats.total.fail_ratio:.2%}")
-    print(f"Average response time: {environment.stats.total.avg_response_time:.0f}ms")
-    print(f"Median response time: {environment.stats.total.median_response_time:.0f}ms")
-    
-    # Save detailed stats
-    if environment.parsed_options and hasattr(environment.parsed_options, 'html_file'):
-        stats_writer = StatsCSVFileWriter(
-            environment,
-            percentiles_to_report=[50, 90, 95, 99]
-        )
-
-
-# Standalone test scenarios for different load patterns
-class StressTestUser(HttpUser):
-    """
-    User for stress testing - generates maximum load.
-    """
-    wait_time = between(0.1, 0.5)  # Minimal wait time
-    
-    def on_start(self):
-        self.client.headers["X-API-Key"] = API_KEY
-    
-    @task
-    def stress_search(self):
-        """Rapid-fire search requests."""
-        payload = {
-            "query": f"stress test {random.randint(1, 1000)}",
-            "engines": ["google"],
-            "max_results": 1,
-            "scrape_content": False,
-            "cache_ttl": 0  # No caching for stress test
-        }
-        
-        self.client.post(
-            "/api/v1/search",
-            json=payload,
-            name="Stress Test"
-        )
-
-
-class SpikeTestUser(HttpUser):
-    """
-    User for spike testing - sudden traffic increases.
-    """
-    wait_time = between(0.5, 1)
-    
-    def on_start(self):
-        self.client.headers["X-API-Key"] = API_KEY
-    
-    @task
-    def spike_search(self):
-        """Burst of requests."""
-        # Simulate spike pattern
-        if random.random() < 0.3:  # 30% chance of burst
-            for _ in range(5):  # Send 5 rapid requests
-                payload = {
-                    "query": f"spike test {time.time()}",
-                    "engines": ["google"],
-                    "max_results": 3,
-                    "scrape_content": False
-                }
-                
-                self.client.post(
-                    "/api/v1/search",
-                    json=payload,
-                    name="Spike Test"
-                )
-                time.sleep(0.1)
-
-
-# Configuration for different test scenarios
-TEST_SCENARIOS = {
-    "normal": {
-        "users": 50,
-        "spawn_rate": 2,
-        "run_time": "5m",
-        "description": "Normal load test"
-    },
-    "stress": {
-        "users": 200,
-        "spawn_rate": 10,
-        "run_time": "10m",
-        "description": "Stress test with high load"
-    },
-    "spike": {
-        "users": 100,
-        "spawn_rate": 50,
-        "run_time": "3m",
-        "description": "Spike test with sudden traffic"
-    },
-    "endurance": {
-        "users": 30,
-        "spawn_rate": 1,
-        "run_time": "60m",
-        "description": "Endurance test over extended period"
-    }
-}
-
-
-if __name__ == "__main__":
-    """
-    Run load test programmatically.
-    Usage: python locustfile.py [scenario]
-    """
-    import sys
-    from locust.env import Environment
-    from locust.stats import stats_printer, stats_history
-    import gevent
-    
-    # Get scenario from command line
-    scenario = sys.argv[1] if len(sys.argv) > 1 else "normal"
-    config = TEST_SCENARIOS.get(scenario, TEST_SCENARIOS["normal"])
-    
-    print(f"\nRunning {config['description']}")
-    print(f"Users: {config['users']}, Spawn rate: {config['spawn_rate']}, Duration: {config['run_time']}\n")
-    
-    # Setup Environment
-    env = Environment(user_classes=[UnSearchUser, MobileUser], host="http://localhost:8000")
-    env.create_local_runner()
-    
-    # Start test
-    env.runner.start(config["users"], spawn_rate=config["spawn_rate"])
-    
-    # Run for specified time
-    duration = int(config["run_time"][:-1]) * (60 if "m" in config["run_time"] else 1)
-    gevent.spawn(stats_printer(env.stats))
-    gevent.spawn(stats_history, env.runner)
-    
-    env.runner.greenlet.join(timeout=duration)
-    env.runner.quit()
-    
-    # Print final stats
-    print("\n" + "="*50)
-    print("Test completed!")
-    print(f"Total requests: {env.stats.total.num_requests}")
-    print(f"Failure rate: {env.stats.total.fail_ratio:.2%}")
-    print(f"Avg response time: {env.stats.total.avg_response_time:.0f}ms")
-    print("="*50)
diff --git a/apps/backend/tests/performance/test_benchmarks.py b/apps/backend/tests/performance/test_benchmarks.py
deleted file mode 100644
index cfd6417..0000000
--- a/apps/backend/tests/performance/test_benchmarks.py
+++ /dev/null
@@ -1,504 +0,0 @@
-"""
-Performance benchmarks for the UnSearch API.
-Uses pytest-benchmark for accurate performance measurements.
-"""
-import asyncio
-import json
-import time
-from typing import List, Dict, Any
-import pytest
-import httpx
-from httpx import AsyncClient
-import statistics
-import random
-import os
-
-from app.services.cache import CacheService
-from app.services.scraping import ContentScrapingService
-from app.services.searxng import SearXNGService
-from app.models.requests import UnSearchRequest, ScrapingConfig
-from app.utils.text_processing import (
-    sanitize_text, extract_snippet, detect_language, calculate_text_quality
-)
-
-# Test data
-SAMPLE_QUERIES = [
-    "Python programming",
-    "machine learning algorithms", 
-    "web scraping techniques",
-    "FastAPI performance optimization",
-    "Docker containerization best practices"
-]
-
-SAMPLE_HTML = """
-
-
-Sample Page
-
-    
Main Title

-    
This is a sample paragraph with some content for testing text extraction and processing. 
-    It contains multiple sentences to test quality scoring.

-    

-        
Section Title

-        
Another paragraph with more content. This helps test the extraction of main content 
-        from HTML pages.

-        
    
-            
  • List item 1
    • 
-            
  • List item 2
    • 
-            
  • List item 3
    • 
-        

-    

-    
Footer content

-
-
-""" * 10  # Make it larger for realistic testing
-
-
-class TestServiceBenchmarks:
-    """Benchmark individual service components."""
-    
-    @pytest.mark.benchmark(group="cache")
-    def test_cache_write_performance(self, benchmark):
-        """Benchmark cache write operations."""
-        cache = CacheService()
-        
-        async def cache_write():
-            await cache.initialize()
-            data = {"results": [{"title": f"Result {i}"} for i in range(100)]}
-            cache_key = f"test_key_{random.randint(1, 1000000)}"
-            await cache.set_search_results(cache_key, data, ttl=3600)
-            await cache.close()
-        
-        def run_cache_write():
-            asyncio.run(cache_write())
-        
-        benchmark(run_cache_write)
-    
-    @pytest.mark.benchmark(group="cache")
-    def test_cache_read_performance(self, benchmark):
-        """Benchmark cache read operations."""
-        cache = CacheService()
-        
-        async def setup():
-            await cache.initialize()
-            # Pre-populate cache
-            for i in range(100):
-                data = {"results": [{"title": f"Result {j}"} for j in range(10)]}
-                await cache.set_search_results(f"test_key_{i}", data, ttl=3600)
-            return cache
-        
-        cache_instance = asyncio.run(setup())
-        
-        async def cache_read():
-            key = f"test_key_{random.randint(0, 99)}"
-            result = await cache_instance.get_search_results(key)
-            return result
-        
-        def run_cache_read():
-            return asyncio.run(cache_read())
-        
-        result = benchmark(run_cache_read)
-        assert result is not None
-        
-        # Cleanup
-        asyncio.run(cache_instance.close())
-    
-    @pytest.mark.benchmark(group="text-processing")
-    def test_text_sanitization_performance(self, benchmark):
-        """Benchmark text sanitization."""
-        sample_text = SAMPLE_HTML * 5
-        
-        result = benchmark(sanitize_text, sample_text)
-        assert len(result) > 0
-    
-    @pytest.mark.benchmark(group="text-processing")
-    def test_snippet_extraction_performance(self, benchmark):
-        """Benchmark snippet extraction."""
-        text = "This is a long text " * 100 + "important keyword here " + "more text " * 100
-        
-        result = benchmark(extract_snippet, text, "keyword", 150)
-        assert "keyword" in result.lower()
-    
-    @pytest.mark.benchmark(group="text-processing")
-    def test_language_detection_performance(self, benchmark):
-        """Benchmark language detection."""
-        texts = [
-            "This is an English text for language detection",
-            "Ceci est un texte français pour la détection de langue",
-            "Dies ist ein deutscher Text zur Spracherkennung",
-            "Este es un texto en español para detección de idioma"
-        ]
-        
-        def detect_all():
-            return [detect_language(text) for text in texts]
-        
-        results = benchmark(detect_all)
-        assert len(results) == len(texts)
-    
-    @pytest.mark.benchmark(group="text-processing")
-    def test_text_quality_scoring_performance(self, benchmark):
-        """Benchmark text quality calculation."""
-        sample_texts = [
-            "Short text",
-            "Medium length text with more words and better structure for testing.",
-            SAMPLE_HTML[:500],
-            SAMPLE_HTML
-        ]
-        
-        def calculate_all():
-            return [calculate_text_quality(text) for text in sample_texts]
-        
-        results = benchmark(calculate_all)
-        assert all(0 <= score <= 1 for score in results)
-
-
-class TestAPIEndpointBenchmarks:
-    """Benchmark API endpoint performance."""
-    
-    @pytest.fixture
-    async def client(self):
-        """Create test client."""
-        async with AsyncClient(
-            base_url="http://localhost:8000",
-            headers={"X-API-Key": os.getenv("BENCHMARK_API_KEY", "test-key")},
-            timeout=30.0
-        ) as client:
-            yield client
-    
-    @pytest.mark.benchmark(group="api", min_rounds=5)
-    @pytest.mark.asyncio
-    async def test_search_endpoint_performance(self, benchmark, client):
-        """Benchmark search endpoint."""
-        async def perform_search():
-            response = await client.post("/api/v1/search", json={
-                "query": random.choice(SAMPLE_QUERIES),
-                "engines": ["google"],
-                "max_results": 5,
-                "scrape_content": False,
-                "cache_ttl": 0  # Disable caching for benchmark
-            })
-            return response
-        
-        # Wrap async function for benchmark
-        def run_search():
-            return asyncio.run(perform_search())
-        
-        response = benchmark(run_search)
-        if response.status_code == 200:
-            assert "results" in response.json()
-    
-    @pytest.mark.benchmark(group="api", min_rounds=3)
-    @pytest.mark.asyncio
-    async def test_search_with_scraping_performance(self, benchmark, client):
-        """Benchmark search with content scraping."""
-        async def perform_search_with_scraping():
-            response = await client.post("/api/v1/search", json={
-                "query": random.choice(SAMPLE_QUERIES),
-                "engines": ["google"],
-                "max_results": 2,
-                "scrape_content": True,
-                "cache_ttl": 0
-            })
-            return response
-        
-        def run_search():
-            return asyncio.run(perform_search_with_scraping())
-        
-        # This will be slower due to scraping
-        benchmark.pedantic(run_search, rounds=3, warmup_rounds=1)
-    
-    @pytest.mark.benchmark(group="api")
-    @pytest.mark.asyncio
-    async def test_health_check_performance(self, benchmark, client):
-        """Benchmark health check endpoint."""
-        async def check_health():
-            response = await client.get("/health")
-            return response
-        
-        def run_health_check():
-            return asyncio.run(check_health())
-        
-        response = benchmark(run_health_check)
-        assert response.status_code == 200
-
-
-class TestConcurrencyBenchmarks:
-    """Benchmark concurrent request handling."""
-    
-    @pytest.mark.benchmark(group="concurrency")
-    @pytest.mark.asyncio
-    async def test_concurrent_searches(self, benchmark):
-        """Benchmark concurrent search requests."""
-        async def perform_concurrent_searches(num_requests: int):
-            async with AsyncClient(
-                base_url="http://localhost:8000",
-                headers={"X-API-Key": os.getenv("BENCHMARK_API_KEY", "test-key")},
-                timeout=30.0
-            ) as client:
-                tasks = []
-                for i in range(num_requests):
-                    task = client.post("/api/v1/search", json={
-                        "query": f"concurrent test {i}",
-                        "engines": ["google"],
-                        "max_results": 3,
-                        "scrape_content": False,
-                        "cache_ttl": 0
-                    })
-                    tasks.append(task)
-                
-                responses = await asyncio.gather(*tasks, return_exceptions=True)
-                
-                successful = sum(1 for r in responses 
-                               if not isinstance(r, Exception) and r.status_code == 200)
-                return successful, len(responses)
-        
-        def run_concurrent():
-            return asyncio.run(perform_concurrent_searches(10))
-        
-        successful, total = benchmark(run_concurrent)
-        assert successful > 0
-    
-    @pytest.mark.benchmark(group="concurrency")
-    @pytest.mark.asyncio
-    async def test_concurrent_scraping(self, benchmark):
-        """Benchmark concurrent scraping operations."""
-        scraper = ContentScrapingService()
-        
-        async def perform_concurrent_scraping():
-            await scraper.initialize()
-            
-            urls = [
-                "https://example.com",
-                "https://httpbin.org/html",
-                "https://www.python.org"
-            ] * 3  # Total 9 URLs
-            
-            config = ScrapingConfig(
-                urls=urls,
-                extract_images=True,
-                extract_links=True
-            )
-            
-            results = await scraper.scrape_urls(urls, config)
-            await scraper.close()
-            return len(results)
-        
-        def run_scraping():
-            return asyncio.run(perform_concurrent_scraping())
-        
-        # Scraping is slow, so fewer rounds
-        benchmark.pedantic(run_scraping, rounds=2, warmup_rounds=1)
-
-
-class TestMemoryBenchmarks:
-    """Benchmark memory usage and efficiency."""
-    
-    @pytest.mark.benchmark(group="memory")
-    def test_large_response_handling(self, benchmark):
-        """Benchmark handling of large responses."""
-        # Simulate large search results
-        large_results = [
-            {
-                "rank": i,
-                "title": f"Result {i} " * 10,
-                "url": f"https://example.com/page{i}",
-                "snippet": "Sample content " * 50,
-                "engine": "google",
-                "scraped_content": {
-                    "text": "Large content " * 1000,
-                    "images": [f"https://example.com/img{j}.jpg" for j in range(50)],
-                    "links": [f"https://example.com/link{j}" for j in range(100)]
-                }
-            }
-            for i in range(100)
-        ]
-        
-        def process_large_response():
-            # Simulate processing
-            json_data = json.dumps(large_results)
-            parsed = json.loads(json_data)
-            # Extract just titles (simulate data extraction)
-            titles = [r["title"] for r in parsed]
-            return len(titles)
-        
-        result = benchmark(process_large_response)
-        assert result == 100
-    
-    @pytest.mark.benchmark(group="memory")
-    def test_cache_memory_efficiency(self, benchmark):
-        """Benchmark cache memory efficiency with compression."""
-        cache = CacheService()
-        
-        async def test_compression():
-            await cache.initialize()
-            
-            # Create large data object
-            large_data = {
-                "results": [
-                    {"content": "x" * 10000} for _ in range(100)
-                ]
-            }
-            
-            # Store with compression
-            await cache.set_search_results("large_key", large_data, ttl=60)
-            
-            # Retrieve
-            retrieved = await cache.get_search_results("large_key")
-            
-            await cache.close()
-            return retrieved is not None
-        
-        def run_test():
-            return asyncio.run(test_compression())
-        
-        result = benchmark(run_test)
-        assert result
-
-
-class TestScalingBenchmarks:
-    """Benchmark API scaling characteristics."""
-    
-    @pytest.mark.benchmark(group="scaling")
-    @pytest.mark.parametrize("num_users", [1, 5, 10, 20])
-    @pytest.mark.asyncio
-    async def test_scaling_with_users(self, benchmark, num_users):
-        """Test how performance scales with number of concurrent users."""
-        async def simulate_users():
-            async with AsyncClient(
-                base_url="http://localhost:8000",
-                headers={"X-API-Key": os.getenv("BENCHMARK_API_KEY", "test-key")},
-                timeout=30.0
-            ) as client:
-                tasks = []
-                for user in range(num_users):
-                    # Each user makes 3 requests
-                    for req in range(3):
-                        task = client.post("/api/v1/search", json={
-                            "query": f"user_{user}_request_{req}",
-                            "engines": ["google"],
-                            "max_results": 5,
-                            "scrape_content": False
-                        })
-                        tasks.append(task)
-                
-                start = time.time()
-                responses = await asyncio.gather(*tasks, return_exceptions=True)
-                duration = time.time() - start
-                
-                successful = sum(1 for r in responses 
-                               if not isinstance(r, Exception) and r.status_code == 200)
-                
-                return {
-                    "duration": duration,
-                    "requests": len(tasks),
-                    "successful": successful,
-                    "rps": len(tasks) / duration if duration > 0 else 0
-                }
-        
-        def run_simulation():
-            return asyncio.run(simulate_users())
-        
-        result = benchmark(run_simulation)
-        print(f"\n{num_users} users: {result['rps']:.2f} req/s, "
-              f"{result['successful']}/{result['requests']} successful")
-
-
-def test_generate_performance_report():
-    """Generate a performance report summary."""
-    report = """
-    ================================================================================
-    UnSearch API Performance Benchmark Report
-    ================================================================================
-    
-    Test Environment:
-    - Python Version: 3.11+
-    - API Version: 1.0.0
-    - Test Date: {}
-    
-    Benchmark Results Summary:
-    
-    1. Cache Performance:
-       - Write Operations: < 10ms average
-       - Read Operations: < 5ms average
-       - Compression Overhead: ~20% time increase, 60% space savings
-    
-    2. Text Processing:
-       - Sanitization: < 50ms for 10KB text
-       - Language Detection: < 100ms per text
-       - Quality Scoring: < 20ms per text
-    
-    3. API Endpoints:
-       - Health Check: < 50ms
-       - Search (no scraping): < 500ms average
-       - Search (with scraping): < 5000ms average
-    
-    4. Concurrency:
-       - 10 concurrent searches: > 90% success rate
-       - 20 concurrent users: > 85% success rate
-    
-    5. Scaling Characteristics:
-       - Linear scaling up to 10 concurrent users
-       - Performance degradation at > 20 concurrent users
-       - Optimal throughput: 50-100 req/s
-    
-    Recommendations:
-    - Enable caching for improved performance
-    - Limit concurrent scraping operations to 10
-    - Use connection pooling for database operations
-    - Implement request queuing for high load scenarios
-    
-    ================================================================================
-    """.format(time.strftime("%Y-%m-%d %H:%M:%S"))
-    
-    print(report)
-    return True
-
-
-# Performance test utilities
-def measure_response_times(num_requests: int = 100) -> Dict[str, Any]:
-    """Measure response time statistics."""
-    async def make_requests():
-        async with AsyncClient(
-            base_url="http://localhost:8000",
-            headers={"X-API-Key": os.getenv("BENCHMARK_API_KEY", "test-key")}
-        ) as client:
-            times = []
-            for i in range(num_requests):
-                start = time.time()
-                response = await client.post("/api/v1/search", json={
-                    "query": f"test query {i}",
-                    "engines": ["google"],
-                    "max_results": 5,
-                    "scrape_content": False
-                })
-                duration = time.time() - start
-                times.append(duration * 1000)  # Convert to ms
-            
-            return {
-                "mean": statistics.mean(times),
-                "median": statistics.median(times),
-                "stdev": statistics.stdev(times) if len(times) > 1 else 0,
-                "min": min(times),
-                "max": max(times),
-                "p95": sorted(times)[int(len(times) * 0.95)],
-                "p99": sorted(times)[int(len(times) * 0.99)]
-            }
-    
-    return asyncio.run(make_requests())
-
-
-if __name__ == "__main__":
-    """Run performance analysis."""
-    print("Running performance analysis...")
-    stats = measure_response_times(50)
-    
-    print("\nResponse Time Statistics (ms):")
-    print(f"  Mean: {stats['mean']:.2f}")
-    print(f"  Median: {stats['median']:.2f}")
-    print(f"  Std Dev: {stats['stdev']:.2f}")
-    print(f"  Min: {stats['min']:.2f}")
-    print(f"  Max: {stats['max']:.2f}")
-    print(f"  P95: {stats['p95']:.2f}")
-    print(f"  P99: {stats['p99']:.2f}")
-    
-    test_generate_performance_report()
diff --git a/apps/backend/tests/performance/test_load.py b/apps/backend/tests/performance/test_load.py
deleted file mode 100644
index 1a4aef6..0000000
--- a/apps/backend/tests/performance/test_load.py
+++ /dev/null
@@ -1,400 +0,0 @@
-"""
-Performance and load tests for the UnSearch API.
-"""
-import pytest
-import asyncio
-import time
-from concurrent.futures import ThreadPoolExecutor
-from unittest.mock import Mock, AsyncMock, patch
-import httpx
-from fastapi.testclient import TestClient
-
-from app.main import app
-from app.models.responses import SearchResult
-
-
-@pytest.fixture
-def client():
-    """Create test client."""
-    return TestClient(app)
-
-
-@pytest.fixture
-def mock_fast_services():
-    """Mock services with fast responses for load testing."""
-    with patch('app.services.searxng.get_searxng_service') as mock_searxng, \
-         patch('app.services.scraping.get_scraping_service') as mock_scraper, \
-         patch('app.services.cache.get_cache_service') as mock_cache, \
-         patch('app.services.database.get_database_service') as mock_db:
-        
-        # Mock SearXNG with fast response
-        searxng_mock = AsyncMock()
-        searxng_mock.search = AsyncMock(return_value=[
-            SearchResult(
-                rank=i,
-                title=f"Test Result {i}",
-                url=f"https://example{i}.com",
-                snippet=f"Test snippet {i}",
-                engine="google"
-            ) for i in range(1, 11)
-        ])
-        mock_searxng.return_value = searxng_mock
-        
-        # Mock other services
-        scraper_mock = AsyncMock()
-        scraper_mock.scrape_urls = AsyncMock(return_value=[])
-        mock_scraper.return_value = scraper_mock
-        
-        cache_mock = AsyncMock()
-        cache_mock.get_search_results = AsyncMock(return_value=None)
-        cache_mock.set_search_results = AsyncMock()
-        cache_mock.generate_cache_key = Mock(return_value="test-cache-key")
-        mock_cache.return_value = cache_mock
-        
-        db_mock = AsyncMock()
-        db_mock.get_api_key = AsyncMock(return_value=None)
-        db_mock.log_search_request = AsyncMock()
-        db_mock.log_error = AsyncMock()
-        mock_db.return_value = db_mock
-        
-        yield {
-            'searxng': searxng_mock,
-            'scraper': scraper_mock,
-            'cache': cache_mock,
-            'db': db_mock
-        }
-
-
-class TestPerformance:
-    """Test API performance."""
-    
-    def test_single_request_latency(self, client, mock_fast_services):
-        """Test single request latency."""
-        with patch('app.config.get_settings') as mock_settings:
-            from app.config import get_settings
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            start_time = time.time()
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "performance test",
-                "engines": ["google"],
-                "max_results": 10,
-                "scrape_content": False
-            })
-            
-            end_time = time.time()
-            latency = (end_time - start_time) * 1000  # Convert to ms
-            
-            assert response.status_code == 200
-            assert latency < 1000  # Should respond within 1 second
-            
-            # Check response time header
-            response_time = float(response.headers.get("X-Response-Time", "0"))
-            assert response_time > 0
-    
-    def test_concurrent_requests(self, client, mock_fast_services):
-        """Test concurrent request handling."""
-        with patch('app.config.get_settings') as mock_settings:
-            from app.config import get_settings
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            def make_request(query_num):
-                """Make a single request."""
-                return client.post("/api/v1/search/", json={
-                    "query": f"concurrent test {query_num}",
-                    "engines": ["google"],
-                    "max_results": 5,
-                    "scrape_content": False
-                })
-            
-            # Test with 10 concurrent requests
-            concurrent_requests = 10
-            start_time = time.time()
-            
-            with ThreadPoolExecutor(max_workers=concurrent_requests) as executor:
-                futures = [executor.submit(make_request, i) for i in range(concurrent_requests)]
-                responses = [future.result() for future in futures]
-            
-            end_time = time.time()
-            total_time = end_time - start_time
-            
-            # All requests should succeed
-            assert all(response.status_code == 200 for response in responses)
-            
-            # Should handle concurrent requests efficiently
-            assert total_time < 5.0  # All requests should complete within 5 seconds
-            
-            # Calculate requests per second
-            rps = concurrent_requests / total_time
-            assert rps > 2  # Should handle at least 2 requests per second
-    
-    def test_memory_usage(self, client, mock_fast_services):
-        """Test memory usage during requests."""
-        import psutil
-        import os
-        
-        with patch('app.config.get_settings') as mock_settings:
-            from app.config import get_settings
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            process = psutil.Process(os.getpid())
-            initial_memory = process.memory_info().rss
-            
-            # Make multiple requests
-            for i in range(50):
-                response = client.post("/api/v1/search/", json={
-                    "query": f"memory test {i}",
-                    "engines": ["google"],
-                    "max_results": 10,
-                    "scrape_content": False
-                })
-                assert response.status_code == 200
-            
-            final_memory = process.memory_info().rss
-            memory_increase = final_memory - initial_memory
-            
-            # Memory increase should be reasonable (less than 100MB)
-            assert memory_increase < 100 * 1024 * 1024
-    
-    def test_large_response_handling(self, client, mock_fast_services):
-        """Test handling of large responses."""
-        # Mock a large number of results
-        large_results = [
-            SearchResult(
-                rank=i,
-                title=f"Large Test Result {i}" * 10,  # Longer titles
-                url=f"https://example{i}.com/very/long/path/to/test/performance",
-                snippet=f"This is a very long snippet for result {i} " * 20,
-                engine="google"
-            ) for i in range(1, 101)  # 100 results
-        ]
-        
-        mock_fast_services['searxng'].search.return_value = large_results
-        
-        with patch('app.config.get_settings') as mock_settings:
-            from app.config import get_settings
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            start_time = time.time()
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "large response test",
-                "engines": ["google"],
-                "max_results": 100,
-                "scrape_content": False
-            })
-            
-            end_time = time.time()
-            
-            assert response.status_code == 200
-            assert end_time - start_time < 2.0  # Should handle large responses quickly
-            
-            data = response.json()
-            assert len(data["results"]) == 100
-    
-    @pytest.mark.asyncio
-    async def test_async_performance(self, mock_fast_services):
-        """Test async operation performance."""
-        from app.services.searxng import get_searxng_service
-        
-        searxng = await get_searxng_service()
-        
-        # Test multiple concurrent searches
-        async def single_search(query_num):
-            return await searxng.search(
-                query=f"async test {query_num}",
-                engines=["google"],
-                language="en"
-            )
-        
-        start_time = time.time()
-        
-        # Run 20 concurrent searches
-        tasks = [single_search(i) for i in range(20)]
-        results = await asyncio.gather(*tasks)
-        
-        end_time = time.time()
-        
-        assert len(results) == 20
-        assert all(isinstance(result, list) for result in results)
-        assert end_time - start_time < 3.0  # Should complete within 3 seconds
-
-
-class TestStressTest:
-    """Stress tests for the API."""
-    
-    @pytest.mark.slow
-    def test_sustained_load(self, client, mock_fast_services):
-        """Test sustained load over time."""
-        with patch('app.config.get_settings') as mock_settings:
-            from app.config import get_settings
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            success_count = 0
-            error_count = 0
-            total_requests = 100
-            
-            start_time = time.time()
-            
-            for i in range(total_requests):
-                try:
-                    response = client.post("/api/v1/search/", json={
-                        "query": f"stress test {i}",
-                        "engines": ["google"],
-                        "max_results": 5,
-                        "scrape_content": False
-                    })
-                    
-                    if response.status_code == 200:
-                        success_count += 1
-                    else:
-                        error_count += 1
-                        
-                except Exception:
-                    error_count += 1
-                
-                # Small delay to avoid overwhelming
-                time.sleep(0.01)
-            
-            end_time = time.time()
-            total_time = end_time - start_time
-            
-            success_rate = success_count / total_requests
-            rps = total_requests / total_time
-            
-            assert success_rate > 0.95  # 95% success rate
-            assert rps > 5  # At least 5 requests per second
-    
-    @pytest.mark.slow
-    def test_error_recovery(self, client, mock_fast_services):
-        """Test error recovery under load."""
-        with patch('app.config.get_settings') as mock_settings:
-            from app.config import get_settings
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            # Simulate intermittent failures
-            call_count = 0
-            original_search = mock_fast_services['searxng'].search
-            
-            async def failing_search(*args, **kwargs):
-                nonlocal call_count
-                call_count += 1
-                if call_count % 5 == 0:  # Fail every 5th request
-                    raise Exception("Simulated failure")
-                return await original_search(*args, **kwargs)
-            
-            mock_fast_services['searxng'].search = failing_search
-            
-            success_count = 0
-            for i in range(20):
-                try:
-                    response = client.post("/api/v1/search/", json={
-                        "query": f"error recovery test {i}",
-                        "engines": ["google"],
-                        "max_results": 5,
-                        "scrape_content": False
-                    })
-                    
-                    if response.status_code == 200:
-                        success_count += 1
-                        
-                except Exception:
-                    pass
-            
-            # Should recover from errors and continue processing
-            assert success_count > 10  # More than half should succeed
-
-
-class TestCachingPerformance:
-    """Test caching performance."""
-    
-    def test_cache_hit_performance(self, client, mock_fast_services):
-        """Test performance of cache hits."""
-        from app.models.responses import UnSearchResponse, SearchMetadata
-        
-        # Mock cached response
-        cached_response = UnSearchResponse(
-            search_metadata=SearchMetadata(
-                query="cached query",
-                engines_used=["google"],
-                engines_succeeded=["google"],
-                engines_failed=[],
-                total_results_found=10,
-                results_returned=10,
-                search_time_ms=100
-            ),
-            results=[],
-            processing_time_ms=50,
-            cached=True,
-            total_results=10,
-            request_id="cached-123"
-        )
-        
-        mock_fast_services['cache'].get_search_results.return_value = cached_response
-        
-        with patch('app.config.get_settings') as mock_settings:
-            from app.config import get_settings
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            # Measure cache hit performance
-            start_time = time.time()
-            
-            response = client.post("/api/v1/search/", json={
-                "query": "cached query",
-                "engines": ["google"],
-                "cache_ttl": 3600
-            })
-            
-            end_time = time.time()
-            cache_hit_time = (end_time - start_time) * 1000
-            
-            assert response.status_code == 200
-            assert response.json()["cached"] is True
-            assert cache_hit_time < 100  # Cache hits should be very fast
-    
-    def test_cache_performance_comparison(self, client, mock_fast_services):
-        """Compare performance with and without cache."""
-        with patch('app.config.get_settings') as mock_settings:
-            from app.config import get_settings
-            settings = get_settings()
-            settings.api_keys = []
-            mock_settings.return_value = settings
-            
-            # Test without cache
-            start_time = time.time()
-            response1 = client.post("/api/v1/search/", json={
-                "query": "performance comparison",
-                "engines": ["google"],
-                "cache_ttl": 0  # No caching
-            })
-            no_cache_time = time.time() - start_time
-            
-            # Test with cache miss (first request)
-            start_time = time.time()
-            response2 = client.post("/api/v1/search/", json={
-                "query": "performance comparison cached",
-                "engines": ["google"],
-                "cache_ttl": 3600
-            })
-            cache_miss_time = time.time() - start_time
-            
-            assert response1.status_code == 200
-            assert response2.status_code == 200
-            
-            # Cache miss should be similar to no cache (slightly slower due to caching overhead)
-            assert cache_miss_time < no_cache_time * 2
\ No newline at end of file
diff --git a/apps/backend/tests/smoke/test_smoke.py b/apps/backend/tests/smoke/test_smoke.py
deleted file mode 100644
index 715cc7c..0000000
--- a/apps/backend/tests/smoke/test_smoke.py
+++ /dev/null
@@ -1,288 +0,0 @@
-"""
-Smoke tests for quick validation of critical API functionality.
-These tests should run quickly and verify basic operation.
-"""
-import os
-import pytest
-import httpx
-from httpx import AsyncClient
-
-
-BASE_URL = os.getenv("SMOKE_TEST_URL", "http://localhost:8000")
-API_KEY = os.getenv("SMOKE_TEST_API_KEY", "test-api-key")
-
-
-@pytest.mark.asyncio
-class TestSmoke:
-    """Quick smoke tests for critical functionality."""
-    
-    @pytest.fixture
-    async def client(self):
-        """Create HTTP client for tests."""
-        async with AsyncClient(
-            base_url=BASE_URL,
-            timeout=10.0
-        ) as client:
-            yield client
-    
-    async def test_api_is_running(self, client: AsyncClient):
-        """Test that the API is running and responding."""
-        response = await client.get("/")
-        assert response.status_code in [200, 307, 404]  # API is responding
-    
-    async def test_health_check(self, client: AsyncClient):
-        """Test that health endpoint is working."""
-        response = await client.get("/health")
-        assert response.status_code == 200
-        
-        data = response.json()
-        assert data["status"] in ["healthy", "degraded"]
-    
-    async def test_docs_available(self, client: AsyncClient):
-        """Test that API documentation is available."""
-        response = await client.get("/docs")
-        assert response.status_code in [200, 307]  # Docs or redirect to docs
-    
-    async def test_authentication_required(self, client: AsyncClient):
-        """Test that authentication is enforced."""
-        # Try to access protected endpoint without auth
-        response = await client.post("/api/v1/search", json={
-            "query": "test",
-            "engines": ["google"]
-        })
-        
-        # Should require authentication (unless disabled in test env)
-        if response.status_code == 401:
-            assert "X-API-Key" in response.json().get("detail", "").lower() or \
-                   "unauthorized" in response.json().get("message", "").lower()
-    
-    async def test_basic_search(self, client: AsyncClient):
-        """Test basic search functionality."""
-        client.headers["X-API-Key"] = API_KEY
-        
-        response = await client.post("/api/v1/search", json={
-            "query": "Python programming",
-            "engines": ["google"],
-            "max_results": 1,
-            "scrape_content": False  # Quick test without scraping
-        })
-        
-        if response.status_code == 401:
-            pytest.skip("API key not valid for this environment")
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        # Basic structure validation
-        assert "search_metadata" in data
-        assert "results" in data
-        assert isinstance(data["results"], list)
-    
-    async def test_invalid_request_handling(self, client: AsyncClient):
-        """Test that invalid requests are handled properly."""
-        client.headers["X-API-Key"] = API_KEY
-        
-        # Send invalid request (empty query)
-        response = await client.post("/api/v1/search", json={
-            "query": "",
-            "engines": ["google"]
-        })
-        
-        # Should return validation error
-        assert response.status_code in [400, 422]
-        assert "error" in response.json() or "detail" in response.json()
-    
-    async def test_rate_limiting_headers(self, client: AsyncClient):
-        """Test that rate limiting headers are present."""
-        client.headers["X-API-Key"] = API_KEY
-        
-        response = await client.post("/api/v1/search", json={
-            "query": "rate limit test",
-            "engines": ["google"],
-            "max_results": 1
-        })
-        
-        # Check for rate limit headers (if implemented)
-        if "X-RateLimit-Limit" in response.headers:
-            assert "X-RateLimit-Remaining" in response.headers
-            assert "X-RateLimit-Reset" in response.headers
-    
-    async def test_cors_headers(self, client: AsyncClient):
-        """Test CORS headers are properly set."""
-        response = await client.options("/api/v1/search")
-        
-        # Check CORS headers
-        if "Access-Control-Allow-Origin" in response.headers:
-            assert "Access-Control-Allow-Methods" in response.headers
-            assert "Access-Control-Allow-Headers" in response.headers
-    
-    async def test_metrics_endpoint_exists(self, client: AsyncClient):
-        """Test that metrics endpoint exists."""
-        response = await client.get("/metrics")
-        
-        # Metrics might be protected or disabled
-        assert response.status_code in [200, 401, 404]
-        
-        if response.status_code == 200:
-            # Should return Prometheus format
-            assert response.headers.get("content-type", "").startswith("text/plain")
-
-
-@pytest.mark.asyncio
-class TestCriticalPaths:
-    """Test critical user paths quickly."""
-    
-    @pytest.fixture
-    async def auth_client(self):
-        """Create authenticated client."""
-        async with AsyncClient(
-            base_url=BASE_URL,
-            headers={"X-API-Key": API_KEY},
-            timeout=15.0
-        ) as client:
-            yield client
-    
-    async def test_search_to_results_path(self, auth_client: AsyncClient):
-        """Test the critical path from search to getting results."""
-        # Submit search
-        response = await auth_client.post("/api/v1/search", json={
-            "query": "test query",
-            "engines": ["google"],
-            "max_results": 3,
-            "scrape_content": False
-        })
-        
-        if response.status_code == 401:
-            pytest.skip("Authentication not configured for test environment")
-        
-        assert response.status_code == 200
-        data = response.json()
-        
-        # Verify we got results
-        assert len(data["results"]) > 0
-        
-        # Verify result structure
-        first_result = data["results"][0]
-        assert "title" in first_result
-        assert "url" in first_result
-        assert "snippet" in first_result
-    
-    async def test_caching_works(self, auth_client: AsyncClient):
-        """Test that caching is functional."""
-        request_data = {
-            "query": "cache test query",
-            "engines": ["google"],
-            "max_results": 2,
-            "cache_ttl": 60
-        }
-        
-        # First request
-        response1 = await auth_client.post("/api/v1/search", json=request_data)
-        if response1.status_code != 200:
-            pytest.skip("Search not working in test environment")
-        
-        data1 = response1.json()
-        
-        # Second request (should be cached)
-        response2 = await auth_client.post("/api/v1/search", json=request_data)
-        assert response2.status_code == 200
-        
-        data2 = response2.json()
-        
-        # Check cache indicator
-        if "cached" in data2:
-            assert data2["cached"] == True
-        
-        # Results should be identical
-        assert len(data1["results"]) == len(data2["results"])
-    
-    async def test_error_recovery(self, auth_client: AsyncClient):
-        """Test that API recovers from errors gracefully."""
-        # Send request that might cause error
-        response = await auth_client.post("/api/v1/search", json={
-            "query": "test",
-            "engines": ["nonexistent_engine"],
-            "max_results": 1
-        })
-        
-        # Should handle gracefully
-        assert response.status_code in [200, 400, 422]
-        
-        # Try valid request after error
-        response = await auth_client.post("/api/v1/search", json={
-            "query": "recovery test",
-            "engines": ["google"],
-            "max_results": 1
-        })
-        
-        # Should work normally
-        assert response.status_code in [200, 401]
-
-
-def test_environment_configured():
-    """Test that environment is properly configured."""
-    assert BASE_URL, "BASE_URL not configured"
-    assert API_KEY, "API_KEY not configured"
-    
-    # Check URL is valid
-    assert BASE_URL.startswith("http://") or BASE_URL.startswith("https://")
-
-
-if __name__ == "__main__":
-    """Run smoke tests directly."""
-    import sys
-    import asyncio
-    
-    async def run_critical_tests():
-        """Run only the most critical smoke tests."""
-        print(f"Running smoke tests against: {BASE_URL}")
-        
-        async with AsyncClient(base_url=BASE_URL, timeout=10.0) as client:
-            # Test 1: API is running
-            try:
-                response = await client.get("/health")
-                if response.status_code == 200:
-                    print("✅ API is running")
-                else:
-                    print(f"❌ API health check failed: {response.status_code}")
-                    return False
-            except Exception as e:
-                print(f"❌ Cannot reach API: {e}")
-                return False
-            
-            # Test 2: Search works
-            client.headers["X-API-Key"] = API_KEY
-            try:
-                response = await client.post("/api/v1/search", json={
-                    "query": "smoke test",
-                    "engines": ["google"],
-                    "max_results": 1
-                })
-                
-                if response.status_code == 200:
-                    print("✅ Search endpoint works")
-                elif response.status_code == 401:
-                    print("⚠️  Search requires valid authentication")
-                else:
-                    print(f"❌ Search failed: {response.status_code}")
-                    return False
-            except Exception as e:
-                print(f"❌ Search error: {e}")
-                return False
-            
-            # Test 3: Documentation available
-            try:
-                response = await client.get("/docs")
-                if response.status_code in [200, 307]:
-                    print("✅ API documentation available")
-                else:
-                    print("⚠️  API documentation not accessible")
-            except:
-                print("⚠️  Could not check documentation")
-        
-        print("\n✅ All critical smoke tests passed!")
-        return True
-    
-    # Run the tests
-    success = asyncio.run(run_critical_tests())
-    sys.exit(0 if success else 1)
diff --git a/apps/backend/tests/unit/test_models.py b/apps/backend/tests/unit/test_models.py
deleted file mode 100644
index 4918292..0000000
--- a/apps/backend/tests/unit/test_models.py
+++ /dev/null
@@ -1,204 +0,0 @@
-"""
-Unit tests for data models.
-"""
-import pytest
-from pydantic import ValidationError
-
-from app.models.requests import UnSearchRequest, BatchSearchRequest
-from app.models.responses import SearchResult, ScrapedContent, UnSearchResponse
-
-
-class TestUnSearchRequest:
-    """Test UnSearchRequest model."""
-    
-    def test_valid_request(self):
-        """Test creating valid request."""
-        request = UnSearchRequest(
-            query="Python tutorials",
-            engines=["google", "bing"],
-            max_results=10
-        )
-        
-        assert request.query == "Python tutorials"
-        assert request.engines == ["google", "bing"]
-        assert request.max_results == 10
-        assert request.scrape_content is True  # Default
-        assert request.language == "en"  # Default
-        
-    def test_query_validation(self):
-        """Test query validation."""
-        # Empty query
-        with pytest.raises(ValidationError):
-            UnSearchRequest(query="", engines=["google"])
-            
-        # Query too long
-        with pytest.raises(ValidationError):
-            UnSearchRequest(query="x" * 501, engines=["google"])
-            
-    def test_engine_validation(self):
-        """Test engine validation."""
-        # Invalid engine
-        with pytest.raises(ValidationError):
-            UnSearchRequest(
-                query="test",
-                engines=["google", "invalid_engine"]
-            )
-            
-        # Duplicate engines removed
-        request = UnSearchRequest(
-            query="test",
-            engines=["google", "google", "bing"]
-        )
-        assert request.engines == ["google", "bing"]
-        
-    def test_language_validation(self):
-        """Test language code validation."""
-        # Valid language
-        request = UnSearchRequest(
-            query="test",
-            engines=["google"],
-            language="fr"
-        )
-        assert request.language == "fr"
-        
-        # Invalid language format
-        with pytest.raises(ValidationError):
-            UnSearchRequest(
-                query="test",
-                engines=["google"],
-                language="eng"  # Should be 2 letters
-            )
-            
-    def test_async_mode_validation(self):
-        """Test async mode validation."""
-        # Async mode without webhook
-        with pytest.raises(ValidationError):
-            UnSearchRequest(
-                query="test",
-                engines=["google"],
-                async_mode=True,
-                webhook_url=None
-            )
-            
-        # Valid async mode
-        request = UnSearchRequest(
-            query="test",
-            engines=["google"],
-            async_mode=True,
-            webhook_url="https://example.com/webhook"
-        )
-        assert request.async_mode is True
-        assert str(request.webhook_url) == "https://example.com/webhook"
-
-
-class TestBatchSearchRequest:
-    """Test BatchSearchRequest model."""
-    
-    def test_valid_batch_request(self):
-        """Test creating valid batch request."""
-        request = BatchSearchRequest(
-            queries=["Python", "FastAPI", "Web scraping"],
-            engines=["google"],
-            max_results_per_query=5
-        )
-        
-        assert len(request.queries) == 3
-        assert request.max_results_per_query == 5
-        assert request.scrape_content is False  # Default for batch
-        
-    def test_batch_limits(self):
-        """Test batch size limits."""
-        # Too many queries
-        with pytest.raises(ValidationError):
-            BatchSearchRequest(
-                queries=["query"] * 101,  # Max is 100
-                engines=["google"]
-            )
-            
-        # Empty queries
-        with pytest.raises(ValidationError):
-            BatchSearchRequest(
-                queries=[],
-                engines=["google"]
-            )
-
-
-class TestSearchResult:
-    """Test SearchResult model."""
-    
-    def test_valid_result(self):
-        """Test creating valid search result."""
-        result = SearchResult(
-            rank=1,
-            title="Test Result",
-            url="https://example.com",
-            snippet="This is a test snippet",
-            engine="google"
-        )
-        
-        assert result.rank == 1
-        assert result.title == "Test Result"
-        assert str(result.url) == "https://example.com/"
-        assert result.cached is False  # Default
-        
-    def test_with_scraped_content(self):
-        """Test result with scraped content."""
-        scraped = ScrapedContent(
-            url="https://example.com",
-            title="Page Title",
-            text="Page content",
-            extraction_success=True,
-            extraction_time_ms=100,
-            word_count=50,
-            metadata={},
-            content_quality_score=0.8
-        )
-        
-        result = SearchResult(
-            rank=1,
-            title="Test",
-            url="https://example.com",
-            snippet="Test",
-            engine="google",
-            scraped_content=scraped
-        )
-        
-        assert result.scraped_content is not None
-        assert result.scraped_content.extraction_success is True
-
-
-class TestUnSearchResponse:
-    """Test UnSearchResponse model."""
-    
-    def test_response_serialization(self, sample_search_result):
-        """Test response serialization."""
-        from app.models.responses import SearchMetadata
-        
-        metadata = SearchMetadata(
-            query="test",
-            engines_used=["google"],
-            engines_succeeded=["google"],
-            engines_failed=[],
-            total_results_found=10,
-            results_returned=1,
-            search_time_ms=500
-        )
-        
-        response = UnSearchResponse(
-            search_metadata=metadata,
-            results=[SearchResult(**sample_search_result)],
-            processing_time_ms=1000,
-            cached=False,
-            total_results=1,
-            request_id="test-123"
-        )
-        
-        # Test JSON serialization
-        json_data = response.json()
-        assert "search_metadata" in json_data
-        assert "results" in json_data
-        
-        # Test dict conversion
-        dict_data = response.dict()
-        assert dict_data["request_id"] == "test-123"
-        assert dict_data["processing_time_ms"] == 1000
diff --git a/apps/backend/tests/unit/test_services.py b/apps/backend/tests/unit/test_services.py
deleted file mode 100644
index 9c4c03d..0000000
--- a/apps/backend/tests/unit/test_services.py
+++ /dev/null
@@ -1,361 +0,0 @@
-"""
-Unit tests for services.
-"""
-import pytest
-import asyncio
-from unittest.mock import Mock, AsyncMock, patch
-import httpx
-import redis.asyncio as redis
-
-from app.services.searxng import SearXNGService
-from app.services.scraping import ContentScrapingService
-from app.services.cache import CacheService
-from app.services.database import DatabaseService
-from app.models.requests import UnSearchRequest, ScrapingConfig
-from app.models.responses import SearchResult, ServiceHealth
-from app.config import get_settings
-
-
-@pytest.fixture
-def mock_settings():
-    """Mock settings for testing."""
-    settings = get_settings()
-    settings.searxng_url = "http://test-searxng:8080"
-    settings.redis_url = "redis://test-redis:6379"
-    settings.database_url = "postgresql://test:test@test-db:5432/test"
-    return settings
-
-
-class TestSearXNGService:
-    """Test SearXNG service."""
-    
-    @pytest.fixture
-    async def searxng_service(self):
-        """Create SearXNG service for testing."""
-        service = SearXNGService()
-        yield service
-        await service.close()
-    
-    @pytest.mark.asyncio
-    async def test_initialize(self, searxng_service):
-        """Test service initialization."""
-        with patch('httpx.AsyncClient') as mock_client:
-            mock_client.return_value.get = AsyncMock(return_value=Mock(cookies={}))
-            await searxng_service.initialize()
-            assert searxng_service._client is not None
-    
-    @pytest.mark.asyncio
-    async def test_search_success(self, searxng_service):
-        """Test successful search."""
-        mock_response = Mock()
-        mock_response.json.return_value = {
-            "results": [
-                {
-                    "title": "Test Result",
-                    "url": "https://example.com",
-                    "content": "Test content",
-                    "engine": "google"
-                }
-            ]
-        }
-        mock_response.elapsed.total_seconds.return_value = 1.5
-        
-        with patch.object(searxng_service, '_client') as mock_client:
-            mock_client.get = AsyncMock(return_value=mock_response)
-            
-            results = await searxng_service.search(
-                query="test query",
-                engines=["google"],
-                language="en"
-            )
-            
-            assert len(results) == 1
-            assert results[0].title == "Test Result"
-            assert results[0].url == "https://example.com"
-            assert results[0].engine == "google"
-    
-    @pytest.mark.asyncio
-    async def test_search_error_handling(self, searxng_service):
-        """Test search error handling."""
-        with patch.object(searxng_service, '_client') as mock_client:
-            mock_client.get = AsyncMock(side_effect=httpx.HTTPError("Connection failed"))
-            
-            with pytest.raises(httpx.HTTPError):
-                await searxng_service.search(
-                    query="test query",
-                    engines=["google"]
-                )
-    
-    @pytest.mark.asyncio
-    async def test_health_check(self, searxng_service):
-        """Test health check."""
-        mock_response = Mock()
-        mock_response.headers = {"X-SearXNG-Version": "1.0.0"}
-        
-        with patch.object(searxng_service, '_client') as mock_client:
-            mock_client.get = AsyncMock(return_value=mock_response)
-            with patch.object(searxng_service, 'search', return_value=[]):
-                
-                health = await searxng_service.health_check()
-                
-                assert health.status == "healthy"
-                assert health.latency_ms > 0
-    
-    def test_generate_cache_key(self, searxng_service):
-        """Test cache key generation."""
-        key1 = searxng_service.generate_cache_key("test query", ["google"])
-        key2 = searxng_service.generate_cache_key("test query", ["google"])
-        key3 = searxng_service.generate_cache_key("different query", ["google"])
-        
-        assert key1 == key2  # Same inputs should produce same key
-        assert key1 != key3  # Different inputs should produce different keys
-
-
-class TestContentScrapingService:
-    """Test content scraping service."""
-    
-    @pytest.fixture
-    async def scraping_service(self):
-        """Create scraping service for testing."""
-        service = ContentScrapingService()
-        yield service
-        await service.close()
-    
-    @pytest.mark.asyncio
-    async def test_initialize(self, scraping_service):
-        """Test service initialization."""
-        await scraping_service.initialize()
-        assert scraping_service._client is not None
-    
-    @pytest.mark.asyncio
-    async def test_scrape_single_url_success(self, scraping_service):
-        """Test successful URL scraping."""
-        html_content = """
-        
-        Test Page
-        
-            
Test Heading

-            
Test content paragraph.

-            Test image
-            Test link
-        
-        
-        """
-        
-        mock_response = Mock()
-        mock_response.content = html_content.encode('utf-8')
-        mock_response.headers = {"content-type": "text/html; charset=utf-8"}
-        mock_response.raise_for_status = Mock()
-        
-        with patch.object(scraping_service, '_client') as mock_client:
-            mock_client.get = AsyncMock(return_value=mock_response)
-            
-            result = await scraping_service._scrape_url("https://example.com")
-            
-            assert result.extraction_success is True
-            assert result.title == "Test Page"
-            assert "Test content paragraph" in result.text
-            assert result.word_count > 0
-            assert result.content_quality_score > 0
-    
-    @pytest.mark.asyncio
-    async def test_scrape_multiple_urls(self, scraping_service):
-        """Test scraping multiple URLs."""
-        urls = ["https://example1.com", "https://example2.com"]
-        
-        html_content = "
Test content
"
-        mock_response = Mock()
-        mock_response.content = html_content.encode('utf-8')
-        mock_response.headers = {"content-type": "text/html"}
-        mock_response.raise_for_status = Mock()
-        
-        with patch.object(scraping_service, '_client') as mock_client:
-            mock_client.get = AsyncMock(return_value=mock_response)
-            
-            results = await scraping_service.scrape_urls(urls)
-            
-            assert len(results) == 2
-            assert all(result.extraction_success for result in results)
-    
-    @pytest.mark.asyncio
-    async def test_robots_txt_check(self, scraping_service):
-        """Test robots.txt checking."""
-        robots_content = """
-        User-agent: *
-        Disallow: /admin
-        Allow: /
-        """
-        
-        mock_response = Mock()
-        mock_response.status_code = 200
-        mock_response.text = robots_content
-        
-        with patch.object(scraping_service, '_client') as mock_client:
-            mock_client.get = AsyncMock(return_value=mock_response)
-            
-            # Should allow normal pages
-            allowed = await scraping_service._check_robots_txt("https://example.com/page")
-            assert allowed is True
-            
-            # Should disallow admin pages
-            disallowed = await scraping_service._check_robots_txt("https://example.com/admin/secret")
-            assert disallowed is False
-
-
-class TestCacheService:
-    """Test cache service."""
-    
-    @pytest.fixture
-    async def cache_service(self):
-        """Create cache service for testing."""
-        service = CacheService()
-        yield service
-        await service.close()
-    
-    @pytest.mark.asyncio
-    async def test_initialize(self, cache_service):
-        """Test service initialization."""
-        with patch('redis.asyncio.ConnectionPool.from_url') as mock_pool:
-            with patch('redis.asyncio.Redis') as mock_redis:
-                mock_redis.return_value.ping = AsyncMock()
-                await cache_service.initialize()
-                assert cache_service._client is not None
-    
-    @pytest.mark.asyncio
-    async def test_cache_operations(self, cache_service):
-        """Test cache set and get operations."""
-        from app.models.responses import UnSearchResponse, SearchMetadata
-        
-        # Mock Redis client
-        mock_redis = AsyncMock()
-        cache_service._client = mock_redis
-        
-        # Test data
-        response = UnSearchResponse(
-            search_metadata=SearchMetadata(
-                query="test",
-                engines_used=["google"],
-                engines_succeeded=["google"],
-                engines_failed=[],
-                total_results_found=1,
-                results_returned=1,
-                search_time_ms=100
-            ),
-            results=[],
-            processing_time_ms=200,
-            cached=False,
-            total_results=1,
-            request_id="test-123"
-        )
-        
-        # Test cache set
-        await cache_service.set_search_results("test-key", response, 3600)
-        mock_redis.setex.assert_called_once()
-        
-        # Test cache get
-        mock_redis.get = AsyncMock(return_value=b'{"test": "data"}')
-        cached = await cache_service.get_search_results("test-key")
-        # Result will be None due to serialization mocking, but operation should complete
-        mock_redis.get.assert_called_once_with("test-key")
-    
-    def test_generate_cache_key(self, cache_service):
-        """Test cache key generation."""
-        request = UnSearchRequest(
-            query="test query",
-            engines=["google"],
-            max_results=10
-        )
-        
-        key1 = cache_service.generate_cache_key(request)
-        key2 = cache_service.generate_cache_key(request)
-        
-        assert key1 == key2  # Same request should produce same key
-        assert key1.startswith("search:")
-        assert len(key1) > 20  # Should be reasonably long
-
-
-class TestDatabaseService:
-    """Test database service."""
-    
-    @pytest.fixture
-    async def db_service(self):
-        """Create database service for testing."""
-        service = DatabaseService()
-        yield service
-        await service.close()
-    
-    @pytest.mark.asyncio
-    async def test_initialize(self, db_service):
-        """Test service initialization."""
-        with patch.object(db_service.engine, 'begin') as mock_begin:
-            mock_conn = AsyncMock()
-            mock_begin.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
-            mock_begin.return_value.__aexit__ = AsyncMock()
-            mock_conn.run_sync = AsyncMock()
-            
-            await db_service.initialize()
-            mock_begin.assert_called_once()
-    
-    @pytest.mark.asyncio
-    async def test_api_key_operations(self, db_service):
-        """Test API key database operations."""
-        from app.models.database import APIKey
-        
-        # Mock session
-        mock_session = AsyncMock()
-        mock_result = Mock()
-        mock_result.scalar_one_or_none.return_value = APIKey(
-            id=1,
-            key="test-key",
-            name="Test Key",
-            is_active=True
-        )
-        mock_session.execute = AsyncMock(return_value=mock_result)
-        mock_session.commit = AsyncMock()
-        mock_session.close = AsyncMock()
-        
-        with patch.object(db_service, 'get_session') as mock_get_session:
-            mock_get_session.return_value.__aenter__ = AsyncMock(return_value=mock_session)
-            mock_get_session.return_value.__aexit__ = AsyncMock()
-            
-            api_key = await db_service.get_api_key("test-key")
-            assert api_key is not None
-            assert api_key.key == "test-key"
-    
-    @pytest.mark.asyncio
-    async def test_error_logging(self, db_service):
-        """Test error logging."""
-        mock_session = AsyncMock()
-        mock_session.add = Mock()
-        mock_session.commit = AsyncMock()
-        mock_session.close = AsyncMock()
-        
-        with patch.object(db_service, 'get_session') as mock_get_session:
-            mock_get_session.return_value.__aenter__ = AsyncMock(return_value=mock_session)
-            mock_get_session.return_value.__aexit__ = AsyncMock()
-            
-            await db_service.log_error(
-                error_type="TestError",
-                error_message="Test error message",
-                request_id="test-123"
-            )
-            
-            mock_session.add.assert_called_once()
-            mock_session.commit.assert_called_once()
-
-
-@pytest.mark.asyncio
-async def test_service_integration():
-    """Test service integration."""
-    # This would test how services work together
-    # For now, just verify they can be imported and initialized
-    from app.services.searxng import get_searxng_service
-    from app.services.scraping import get_scraping_service
-    from app.services.cache import get_cache_service
-    from app.services.database import get_database_service
-    
-    # These should not raise exceptions
-    assert get_searxng_service is not None
-    assert get_scraping_service is not None
-    assert get_cache_service is not None
-    assert get_database_service is not None
diff --git a/apps/backend/tests/unit/test_utils.py b/apps/backend/tests/unit/test_utils.py
deleted file mode 100644
index def17a9..0000000
--- a/apps/backend/tests/unit/test_utils.py
+++ /dev/null
@@ -1,315 +0,0 @@
-"""
-Unit tests for utility functions.
-"""
-import pytest
-from unittest.mock import Mock, patch
-
-from app.utils.text_processing import (
-    sanitize_text, extract_snippet, detect_language,
-    calculate_text_quality, extract_keywords, truncate_text, normalize_url
-)
-from app.utils.validators import (
-    validate_query, validate_url, validate_engines, validate_language_code,
-    validate_css_selector, validate_custom_selectors, validate_webhook_url
-)
-from app.utils.security import (
-    generate_api_key, hash_password, verify_password, sanitize_input,
-    is_safe_url, generate_csrf_token, verify_csrf_token
-)
-from app.utils.exceptions import (
-    UnSearchException, BadRequestException, UnauthorizedException
-)
-
-
-class TestTextProcessing:
-    """Test text processing utilities."""
-    
-    def test_sanitize_text(self):
-        """Test text sanitization."""
-        # HTML entities
-        assert sanitize_text("Hello & world") == "Hello & world"
-        
-        # HTML tags
-        assert sanitize_text("Hello world") == "Hello world"
-        
-        # Extra whitespace
-        assert sanitize_text("Hello    world\n\n\n") == "Hello world"
-        
-        # Empty input
-        assert sanitize_text("") == ""
-        assert sanitize_text(None) == ""
-    
-    def test_extract_snippet(self):
-        """Test snippet extraction."""
-        text = "This is a test document. It contains multiple sentences about testing. The test should extract relevant content."
-        query = "test"
-        
-        snippet = extract_snippet(text, query, max_length=50)
-        assert len(snippet) <= 50
-        assert "test" in snippet.lower()
-    
-    def test_detect_language(self):
-        """Test language detection."""
-        # English text
-        english_text = "This is a sample English text for language detection testing."
-        lang = detect_language(english_text)
-        assert lang == "en" or lang is None  # langdetect may not work in test env
-        
-        # Short text should return None
-        assert detect_language("Hi") is None
-        
-        # Empty text should return None
-        assert detect_language("") is None
-    
-    def test_calculate_text_quality(self):
-        """Test text quality calculation."""
-        # Good quality text
-        good_text = "This is a well-written article with proper sentence structure. It contains multiple paragraphs and good vocabulary diversity. The content is informative and well-structured."
-        quality = calculate_text_quality(good_text)
-        assert 0.0 <= quality <= 1.0
-        
-        # Poor quality text
-        poor_text = "abc def"
-        quality = calculate_text_quality(poor_text)
-        assert quality < 0.5
-        
-        # Empty text
-        assert calculate_text_quality("") == 0.0
-    
-    def test_extract_keywords(self):
-        """Test keyword extraction."""
-        text = "Python programming language development software engineering code"
-        keywords = extract_keywords(text, max_keywords=5)
-        
-        assert isinstance(keywords, list)
-        assert len(keywords) <= 5
-        assert all(isinstance(word, str) for word in keywords)
-    
-    def test_truncate_text(self):
-        """Test text truncation."""
-        text = "This is a long text that needs to be truncated at word boundaries"
-        
-        truncated = truncate_text(text, 20)
-        assert len(truncated) <= 20
-        assert truncated.endswith("...")
-        
-        # Short text should not be truncated
-        short_text = "Short"
-        assert truncate_text(short_text, 20) == short_text
-    
-    def test_normalize_url(self):
-        """Test URL normalization."""
-        # Remove tracking parameters
-        url = "https://example.com/page?utm_source=test&utm_medium=email&normal_param=value"
-        normalized = normalize_url(url)
-        assert "utm_source" not in normalized
-        assert "utm_medium" not in normalized
-        assert "normal_param=value" in normalized
-        
-        # Remove trailing slashes
-        assert normalize_url("https://example.com/") == "https://example.com"
-
-
-class TestValidators:
-    """Test validation utilities."""
-    
-    def test_validate_query(self):
-        """Test query validation."""
-        # Valid query
-        assert validate_query("python programming") == "python programming"
-        
-        # Empty query should raise error
-        with pytest.raises(ValueError):
-            validate_query("")
-        
-        # Too long query should raise error
-        with pytest.raises(ValueError):
-            validate_query("x" * 501)
-        
-        # Suspicious content should raise error
-        with pytest.raises(ValueError):
-            validate_query("")
-    
-    def test_validate_url(self):
-        """Test URL validation."""
-        # Valid URLs
-        assert validate_url("https://example.com") == "https://example.com"
-        assert validate_url("http://test.org/path") == "http://test.org/path"
-        
-        # Invalid URLs should raise errors
-        with pytest.raises(ValueError):
-            validate_url("")
-        
-        with pytest.raises(ValueError):
-            validate_url("ftp://example.com")
-        
-        with pytest.raises(ValueError):
-            validate_url("https://127.0.0.1")  # Private IP
-        
-        # Allow private IPs when specified
-        assert validate_url("https://127.0.0.1", allow_private=True)
-    
-    def test_validate_engines(self):
-        """Test engines validation."""
-        # Valid engines
-        engines = validate_engines(["google", "bing"])
-        assert "google" in engines
-        assert "bing" in engines
-        
-        # Remove duplicates
-        engines = validate_engines(["google", "google", "bing"])
-        assert len(engines) == 2
-        
-        # Invalid engine should raise error
-        with pytest.raises(ValueError):
-            validate_engines(["invalid_engine"])
-        
-        # Empty list should raise error
-        with pytest.raises(ValueError):
-            validate_engines([])
-    
-    def test_validate_language_code(self):
-        """Test language code validation."""
-        # Valid codes
-        assert validate_language_code("en") == "en"
-        assert validate_language_code("ES") == "es"  # Case insensitive
-        
-        # Invalid format should raise error
-        with pytest.raises(ValueError):
-            validate_language_code("eng")
-        
-        with pytest.raises(ValueError):
-            validate_language_code("1")
-    
-    def test_validate_css_selector(self):
-        """Test CSS selector validation."""
-        # Valid selectors
-        assert validate_css_selector("div.class") == "div.class"
-        assert validate_css_selector("#id") == "#id"
-        assert validate_css_selector("div > p") == "div > p"
-        
-        # Empty selector should raise error
-        with pytest.raises(ValueError):
-            validate_css_selector("")
-        
-        # Dangerous content should raise error
-        with pytest.raises(ValueError):
-            validate_css_selector("javascript:alert()")
-    
-    def test_validate_custom_selectors(self):
-        """Test custom selectors validation."""
-        selectors = {
-            "title": "h1",
-            "content": "div.content",
-            "author": ".author"
-        }
-        
-        validated = validate_custom_selectors(selectors)
-        assert len(validated) == 3
-        assert validated["title"] == "h1"
-        
-        # Invalid field name should raise error
-        with pytest.raises(ValueError):
-            validate_custom_selectors({"invalid-field": "div"})
-    
-    def test_validate_webhook_url(self):
-        """Test webhook URL validation."""
-        # Valid webhook URL
-        url = validate_webhook_url("https://api.example.com/webhook")
-        assert url == "https://api.example.com/webhook"
-        
-        # Invalid URL should raise error
-        with pytest.raises(ValueError):
-            validate_webhook_url("invalid-url")
-
-
-class TestSecurity:
-    """Test security utilities."""
-    
-    def test_generate_api_key(self):
-        """Test API key generation."""
-        key = generate_api_key()
-        assert isinstance(key, str)
-        assert len(key) > 20
-        
-        # Different calls should produce different keys
-        key2 = generate_api_key()
-        assert key != key2
-    
-    def test_password_hashing(self):
-        """Test password hashing and verification."""
-        password = "test_password_123"
-        
-        # Hash password
-        hashed, salt = hash_password(password)
-        assert isinstance(hashed, str)
-        assert isinstance(salt, str)
-        assert hashed != password
-        
-        # Verify correct password
-        assert verify_password(password, hashed, salt) is True
-        
-        # Verify incorrect password
-        assert verify_password("wrong_password", hashed, salt) is False
-    
-    def test_sanitize_input(self):
-        """Test input sanitization."""
-        # Remove dangerous characters
-        dangerous = ""
-        sanitized = sanitize_input(dangerous)
-        assert "