feat: offload CPU-bound HTML parsing and BeautifulSoup to thread pool

backend/app/services/citation_store.py

@@ -12,6 +12,7 @@
 - "r2": Cloudflare R2 bucket via boto3 S3-compatible API
 """
 import asyncio
+import concurrent.futures
 import hashlib
 import json
 import os
@@ -31,6 +32,12 @@
 
 logger = structlog.get_logger(__name__)
 
+# Dedicated thread pool executor for CPU-bound citation content normalization.
+_citation_cpu_executor = concurrent.futures.ThreadPoolExecutor(
+    max_workers=16,
+    thread_name_prefix="citation_cpu"
+)
+
 
 def _detect_content_type(content_bytes: bytes, headers: Optional[Dict[str, str]]) -> str:
     """Best-effort content-type detection."""
@@ -289,15 +296,18 @@ async def store_snapshot(
     ) -> SnapshotBundle:
         """Normalize content, persist bundle, return bundle metadata."""
         fetched_at = fetched_at or datetime.utcnow()
-        bundle = _build_bundle(
-            url=url,
-            raw_bytes=raw_bytes,
-            content_type=content_type,
-            headers=headers or {},
-            engine=engine,
-            fetched_at=fetched_at,
-            request_id=request_id,
-            redirects=redirects,
+        loop = asyncio.get_running_loop()
+        bundle = await loop.run_in_executor(
+            _citation_cpu_executor,
+            _build_bundle,
+            url,
+            raw_bytes,
+            content_type,
+            headers or {},
+            engine,
+            fetched_at,
+            request_id,
+            redirects,
         )
         return await self._persist_bundle(bundle)
 

backend/app/services/scraping/enhanced_scraping.py

@@ -18,6 +18,29 @@
 
 import structlog
 from bs4 import BeautifulSoup
+from app.services.scraping.scraping import _scraping_cpu_executor
+
+def _sync_enhanced_extract(
+    final_content: str,
+    url: str,
+    config: Optional[Any],
+    service_instance: Any
+) -> Dict[str, Any]:
+    soup = BeautifulSoup(final_content, 'lxml')
+    metadata = service_instance._sync_extract_metadata(soup, url)
+    images = service_instance._extract_images(soup, url) if getattr(config, 'extract_images', True) else []
+    links = service_instance._extract_links(soup, url) if getattr(config, 'extract_links', True) else []
+    for element in soup(['script', 'style', 'noscript']):
+        element.decompose()
+    text = sanitize_text(soup.get_text())
+    title = soup.find('title').get_text() if soup.find('title') else metadata.title
+    return {
+        "metadata": metadata,
+        "images": images,
+        "links": links,
+        "text": text,
+        "title": title
+    }
 
 from app.config import get_settings
 from app.models.responses import ScrapedContent, ContentMetadata
@@ -206,24 +229,25 @@ async def _scrape_with_virtual_scrolling(
             )
 
             if scroll_result.success:
-                # Convert virtual scroll result to ScrapedContent
-                soup = BeautifulSoup(scroll_result.final_content, 'lxml')
-
-                # Extract metadata
-                metadata = await super().extract_metadata(soup, url)
-
-                # Extract images and links
-                images = super()._extract_images(soup, url) if getattr(config, 'extract_images', True) else []
-                links = super()._extract_links(soup, url) if getattr(config, 'extract_links', True) else []
-
-                # Get clean text
-                for element in soup(['script', 'style', 'noscript']):
-                    element.decompose()
-                text = sanitize_text(soup.get_text())
+                # Convert virtual scroll result to ScrapedContent in executor
+                loop = asyncio.get_running_loop()
+                extracted = await loop.run_in_executor(
+                    _scraping_cpu_executor,
+                    _sync_enhanced_extract,
+                    scroll_result.final_content,
+                    url,
+                    config,
+                    self
+                )
+                metadata = extracted["metadata"]
+                images = extracted["images"]
+                links = extracted["links"]
+                text = extracted["text"]
+                title = extracted["title"]
 
                 return ScrapedContent(
                     url=url,
-                    title=soup.find('title').get_text() if soup.find('title') else metadata.title,
+                    title=title,
                     text=text,
                     html=scroll_result.final_content,
                     images=images,
@@ -499,8 +523,12 @@ async def _combine_enhanced_results(
 
             # Update text with filtered content if using markdown output
             if getattr(config, 'response_format', 'json') == 'markdown':
-                soup = BeautifulSoup(filtered_content, 'lxml')
-                enhanced_data['text'] = sanitize_text(soup.get_text())
+                loop = asyncio.get_running_loop()
+                text_from_filtered = await loop.run_in_executor(
+                    _scraping_cpu_executor,
+                    lambda: sanitize_text(BeautifulSoup(filtered_content, 'lxml').get_text())
+                )
+                enhanced_data['text'] = text_from_filtered
 
         # Add markdown results
         if markdown_result:

backend/app/services/scraping/multi_engine_scraper.py

@@ -5,6 +5,7 @@
 """
 
 import asyncio
+import concurrent.futures
 import time
 from typing import Dict, List, Optional, Any, Union, Tuple
 from enum import Enum
@@ -22,6 +23,31 @@
 logger = structlog.get_logger(__name__)
 settings = get_settings()
 
+# Dedicated thread pool executor for CPU-bound scraper parsing in multi-engine scraper.
+# Avoids exhausting the global default thread pool executor.
+_multi_scraper_cpu_executor = concurrent.futures.ThreadPoolExecutor(
+    max_workers=32,
+    thread_name_prefix="multi_scraper_cpu"
+)
+
+
+def _sync_fast_scrape_parse(html_text: str) -> Tuple[str, str, str, float]:
+    from bs4 import BeautifulSoup
+    soup = BeautifulSoup(html_text, 'lxml')
+
+    # Clean content
+    for element in soup(['script', 'style', 'noscript']):
+        element.decompose()
+
+    text = sanitize_text(soup.get_text())
+    title = soup.find('title').get_text() if soup.find('title') else ""
+
+    # Text metrics are also CPU-bound, run them here
+    lang = detect_language(text)
+    quality = calculate_text_quality(text)
+
+    return title, text, lang, quality
+
 
 class EngineType(Enum):
     """Available scraping engines."""
@@ -182,16 +208,12 @@ async def _fast_scrape(self, request: ScrapeRequest) -> ScrapedContent:
         response = await self.client.get(request.url)
         response.raise_for_status()
 
-        # Basic content extraction
-        from bs4 import BeautifulSoup
-        soup = BeautifulSoup(response.text, 'lxml')
-
-        # Clean content
-        for element in soup(['script', 'style', 'noscript']):
-            element.decompose()
-
-        text = sanitize_text(soup.get_text())
-        title = soup.find('title').get_text() if soup.find('title') else ""
+        loop = asyncio.get_running_loop()
+        title, text, lang, quality = await loop.run_in_executor(
+            _multi_scraper_cpu_executor,
+            _sync_fast_scrape_parse,
+            response.text
+        )
 
         return ScrapedContent(
             url=request.url,
@@ -200,8 +222,8 @@ async def _fast_scrape(self, request: ScrapeRequest) -> ScrapedContent:
             html=response.text,
             extraction_success=True,
             word_count=len(text.split()) if text else 0,
-            language_detected=detect_language(text),
-            content_quality_score=calculate_text_quality(text),
+            language_detected=lang,
+            content_quality_score=quality,
             metadata=ContentMetadata()
         )