Article Heading
+This is the first paragraph of the main article content.
+This is the second paragraph of the main article content.
+diff --git a/backend/app/services/citation_store.py b/backend/app/services/citation_store.py index 698775a..f6e664a 100644 --- a/backend/app/services/citation_store.py +++ b/backend/app/services/citation_store.py @@ -12,6 +12,7 @@ - "r2": Cloudflare R2 bucket via boto3 S3-compatible API """ import asyncio +import concurrent.futures import hashlib import json import os @@ -31,6 +32,12 @@ logger = structlog.get_logger(__name__) +# Dedicated thread pool executor for CPU-bound citation content normalization. +_citation_cpu_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=16, + thread_name_prefix="citation_cpu" +) + def _detect_content_type(content_bytes: bytes, headers: Optional[Dict[str, str]]) -> str: """Best-effort content-type detection.""" @@ -289,15 +296,18 @@ async def store_snapshot( ) -> SnapshotBundle: """Normalize content, persist bundle, return bundle metadata.""" fetched_at = fetched_at or datetime.utcnow() - bundle = _build_bundle( - url=url, - raw_bytes=raw_bytes, - content_type=content_type, - headers=headers or {}, - engine=engine, - fetched_at=fetched_at, - request_id=request_id, - redirects=redirects, + loop = asyncio.get_running_loop() + bundle = await loop.run_in_executor( + _citation_cpu_executor, + _build_bundle, + url, + raw_bytes, + content_type, + headers or {}, + engine, + fetched_at, + request_id, + redirects, ) return await self._persist_bundle(bundle) diff --git a/backend/app/services/scraping/enhanced_scraping.py b/backend/app/services/scraping/enhanced_scraping.py index 5f569c1..eae5257 100644 --- a/backend/app/services/scraping/enhanced_scraping.py +++ b/backend/app/services/scraping/enhanced_scraping.py @@ -18,6 +18,29 @@ import structlog from bs4 import BeautifulSoup +from app.services.scraping.scraping import _scraping_cpu_executor + +def _sync_enhanced_extract( + final_content: str, + url: str, + config: Optional[Any], + service_instance: Any +) -> Dict[str, Any]: + soup = BeautifulSoup(final_content, 'lxml') + metadata = service_instance._sync_extract_metadata(soup, url) + images = service_instance._extract_images(soup, url) if getattr(config, 'extract_images', True) else [] + links = service_instance._extract_links(soup, url) if getattr(config, 'extract_links', True) else [] + for element in soup(['script', 'style', 'noscript']): + element.decompose() + text = sanitize_text(soup.get_text()) + title = soup.find('title').get_text() if soup.find('title') else metadata.title + return { + "metadata": metadata, + "images": images, + "links": links, + "text": text, + "title": title + } from app.config import get_settings from app.models.responses import ScrapedContent, ContentMetadata @@ -206,24 +229,25 @@ async def _scrape_with_virtual_scrolling( ) if scroll_result.success: - # Convert virtual scroll result to ScrapedContent - soup = BeautifulSoup(scroll_result.final_content, 'lxml') - - # Extract metadata - metadata = await super().extract_metadata(soup, url) - - # Extract images and links - images = super()._extract_images(soup, url) if getattr(config, 'extract_images', True) else [] - links = super()._extract_links(soup, url) if getattr(config, 'extract_links', True) else [] - - # Get clean text - for element in soup(['script', 'style', 'noscript']): - element.decompose() - text = sanitize_text(soup.get_text()) + # Convert virtual scroll result to ScrapedContent in executor + loop = asyncio.get_running_loop() + extracted = await loop.run_in_executor( + _scraping_cpu_executor, + _sync_enhanced_extract, + scroll_result.final_content, + url, + config, + self + ) + metadata = extracted["metadata"] + images = extracted["images"] + links = extracted["links"] + text = extracted["text"] + title = extracted["title"] return ScrapedContent( url=url, - title=soup.find('title').get_text() if soup.find('title') else metadata.title, + title=title, text=text, html=scroll_result.final_content, images=images, @@ -499,8 +523,12 @@ async def _combine_enhanced_results( # Update text with filtered content if using markdown output if getattr(config, 'response_format', 'json') == 'markdown': - soup = BeautifulSoup(filtered_content, 'lxml') - enhanced_data['text'] = sanitize_text(soup.get_text()) + loop = asyncio.get_running_loop() + text_from_filtered = await loop.run_in_executor( + _scraping_cpu_executor, + lambda: sanitize_text(BeautifulSoup(filtered_content, 'lxml').get_text()) + ) + enhanced_data['text'] = text_from_filtered # Add markdown results if markdown_result: diff --git a/backend/app/services/scraping/multi_engine_scraper.py b/backend/app/services/scraping/multi_engine_scraper.py index 5f48928..7336707 100644 --- a/backend/app/services/scraping/multi_engine_scraper.py +++ b/backend/app/services/scraping/multi_engine_scraper.py @@ -5,6 +5,7 @@ """ import asyncio +import concurrent.futures import time from typing import Dict, List, Optional, Any, Union, Tuple from enum import Enum @@ -22,6 +23,31 @@ logger = structlog.get_logger(__name__) settings = get_settings() +# Dedicated thread pool executor for CPU-bound scraper parsing in multi-engine scraper. +# Avoids exhausting the global default thread pool executor. +_multi_scraper_cpu_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=32, + thread_name_prefix="multi_scraper_cpu" +) + + +def _sync_fast_scrape_parse(html_text: str) -> Tuple[str, str, str, float]: + from bs4 import BeautifulSoup + soup = BeautifulSoup(html_text, 'lxml') + + # Clean content + for element in soup(['script', 'style', 'noscript']): + element.decompose() + + text = sanitize_text(soup.get_text()) + title = soup.find('title').get_text() if soup.find('title') else "" + + # Text metrics are also CPU-bound, run them here + lang = detect_language(text) + quality = calculate_text_quality(text) + + return title, text, lang, quality + class EngineType(Enum): """Available scraping engines.""" @@ -182,16 +208,12 @@ async def _fast_scrape(self, request: ScrapeRequest) -> ScrapedContent: response = await self.client.get(request.url) response.raise_for_status() - # Basic content extraction - from bs4 import BeautifulSoup - soup = BeautifulSoup(response.text, 'lxml') - - # Clean content - for element in soup(['script', 'style', 'noscript']): - element.decompose() - - text = sanitize_text(soup.get_text()) - title = soup.find('title').get_text() if soup.find('title') else "" + loop = asyncio.get_running_loop() + title, text, lang, quality = await loop.run_in_executor( + _multi_scraper_cpu_executor, + _sync_fast_scrape_parse, + response.text + ) return ScrapedContent( url=request.url, @@ -200,8 +222,8 @@ async def _fast_scrape(self, request: ScrapeRequest) -> ScrapedContent: html=response.text, extraction_success=True, word_count=len(text.split()) if text else 0, - language_detected=detect_language(text), - content_quality_score=calculate_text_quality(text), + language_detected=lang, + content_quality_score=quality, metadata=ContentMetadata() ) diff --git a/backend/app/services/scraping/scraping.py b/backend/app/services/scraping/scraping.py index 84d6c26..817d1af 100644 --- a/backend/app/services/scraping/scraping.py +++ b/backend/app/services/scraping/scraping.py @@ -31,6 +31,25 @@ logger = structlog.get_logger(__name__) settings = get_settings() +import concurrent.futures + +# Dedicated thread pool executor for CPU-bound scraping and parsing. +_scraping_cpu_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=32, + thread_name_prefix="scraping_cpu" +) + +def _sync_extract_title_from_html(html_text: str) -> str: + try: + from bs4 import BeautifulSoup + s = BeautifulSoup(html_text, 'lxml') + t = s.find('title') + if t: + return sanitize_text(t.get_text()) + except Exception: + pass + return "" + class ContentScrapingService: """Service for web content extraction using BeautifulSoup4.""" @@ -277,58 +296,62 @@ async def _scrape_url( except Exception as e: logger.warning("url_cache_write_failed", url=url, error=str(e)) - # Parse HTML - soup = BeautifulSoup(html_content, 'lxml') + # Parse HTML and extract content in executor + loop = asyncio.get_running_loop() + extracted_data = await loop.run_in_executor( + _scraping_cpu_executor, + self._sync_parse_and_extract_all, + html_content, + url, + config + ) - # Remove script and style elements - for element in soup(['script', 'style', 'noscript']): - element.decompose() - - # Extract content based on configuration - if config and config.selectors: - extracted = await self._extract_with_selectors(soup, config.selectors) - else: - extracted = await self._extract_main_content(soup, url) - - # Extract metadata - metadata = await self.extract_metadata(soup, url) + title = extracted_data["title"] + extracted_text = extracted_data["text"] + images = extracted_data["images"] + links = extracted_data["links"] + metadata = extracted_data["metadata"] - # Extract images if requested - images = [] - if not config or config.extract_images: - images = self._extract_images(soup, url) - # Extract links if requested - links = [] - if not config or config.extract_links: - links = self._extract_links(soup, url) - # Optional link head enrichment and scoring - if config and getattr(config, 'link_head', False) and links: - try: - links = await self._enrich_and_score_links(links, config) - except Exception as e: - logger.warning("link_enrichment_failed", url=url, error=str(e)) + if config and getattr(config, 'link_head', False) and links: + try: + links = await self._enrich_and_score_links(links, config) + except Exception as e: + logger.warning("link_enrichment_failed", url=url, error=str(e)) # Detect language - language = detect_language(extracted['text']) + language = await loop.run_in_executor( + _scraping_cpu_executor, + detect_language, + extracted_text + ) # Calculate quality score - quality_score = calculate_text_quality(extracted['text']) + quality_score = await loop.run_in_executor( + _scraping_cpu_executor, + calculate_text_quality, + extracted_text + ) # Calculate processing time extraction_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) # Optionally return Markdown format when requested via config.response_format if config and getattr(config, 'response_format', 'json') == 'markdown': - markdown_text = self._to_markdown(html_content, base_url=str(url)) + markdown_text = await loop.run_in_executor( + _scraping_cpu_executor, + self._to_markdown, + html_content, + str(url) + ) # Overwrite text with markdown output for markdown mode text_out = markdown_text else: - text_out = extracted['text'] + text_out = extracted_text return ScrapedContent( url=url, - title=extracted.get('title', metadata.title), + title=title, text=text_out, html=html_content if config and hasattr(config, 'include_html') and config.include_html else None, images=images, @@ -386,46 +409,56 @@ async def _fetch_with_puppeteer(self, url: str, config: ScrapingConfig) -> Optio async def _scrape_page_payload(self, url: str, html: str, config: ScrapingConfig, page_meta: Optional[Dict[str, Any]] = None) -> ScrapedContent: """Scrape using provided HTML payload (from Puppeteer).""" start_time = asyncio.get_event_loop().time() - soup = BeautifulSoup(html or "", 'lxml') - for element in soup(['script', 'style', 'noscript']): - element.decompose() - - if config and config.selectors: - extracted = await self._extract_with_selectors(soup, config.selectors) - else: - extracted = await self._extract_main_content(soup, url) - - metadata = await self.extract_metadata(soup, url) - - images = [] - if not config or config.extract_images: - images = self._extract_images(soup, url) - - links = [] - if not config or config.extract_links: - links = self._extract_links(soup, url) - # Optional link head enrichment and scoring - if getattr(config, 'link_head', False) and links: - try: - links = await self._enrich_and_score_links(links, config) - except Exception as e: - logger.warning("link_enrichment_failed", url=url, error=str(e)) + loop = asyncio.get_running_loop() + extracted_data = await loop.run_in_executor( + _scraping_cpu_executor, + self._sync_parse_and_extract_all, + html, + url, + config + ) + + title = extracted_data["title"] + extracted_text = extracted_data["text"] + images = extracted_data["images"] + links = extracted_data["links"] + metadata = extracted_data["metadata"] + + # Optional link head enrichment and scoring + if getattr(config, 'link_head', False) and links: + try: + links = await self._enrich_and_score_links(links, config) + except Exception as e: + logger.warning("link_enrichment_failed", url=url, error=str(e)) # Optionally return Markdown if getattr(config, 'response_format', 'json') == 'markdown': - markdown_text = self._to_markdown(html, base_url=str(url)) + markdown_text = await loop.run_in_executor( + _scraping_cpu_executor, + self._to_markdown, + html, + str(url) + ) text_out = markdown_text else: - text_out = extracted['text'] + text_out = extracted_text - language = detect_language(text_out) - quality_score = calculate_text_quality(text_out) + language = await loop.run_in_executor( + _scraping_cpu_executor, + detect_language, + text_out + ) + quality_score = await loop.run_in_executor( + _scraping_cpu_executor, + calculate_text_quality, + text_out + ) extraction_time_ms = int((asyncio.get_event_loop().time() - start_time) * 1000) html_out = html if config and getattr(config, 'include_html', False) else None return ScrapedContent( url=url, - title=extracted.get('title', metadata.title), + title=title, text=text_out, html=html_out, images=images, @@ -479,10 +512,12 @@ async def fetch_title(u: str) -> Dict[str, Any]: title_text = "" if ok and resp.headers.get("content-type", "").startswith("text/html"): try: - s = BeautifulSoup(resp.text, 'lxml') - t = s.find('title') - if t: - title_text = sanitize_text(t.get_text()) + loop = asyncio.get_running_loop() + title_text = await loop.run_in_executor( + _scraping_cpu_executor, + _sync_extract_title_from_html, + resp.text + ) except Exception: title_text = "" return {"url": u, "ok": ok, "title": title_text} @@ -514,26 +549,51 @@ def score(item: Dict[str, Any]) -> float: scored.sort(key=lambda x: (-x[0], x[1])) return [u for _, u in scored] - async def _extract_main_content(self, soup: BeautifulSoup, url: str) -> Dict[str, str]: + def _sync_parse_and_extract_all( + self, + html_content: str, + url: str, + config: Optional[ScrapingConfig] + ) -> Dict[str, Any]: + soup = BeautifulSoup(html_content, 'lxml') + + # Extract metadata first, before decomposing script and style elements + metadata = self._sync_extract_metadata(soup, url) + + # Remove script and style elements + for element in soup(['script', 'style', 'noscript']): + element.decompose() + + # Extract content based on configuration + if config and config.selectors: + extracted = self._sync_extract_with_selectors(soup, config.selectors) + else: + extracted = self._sync_extract_main_content(soup, url) + + # Extract images if requested + images = [] + if not config or config.extract_images: + images = self._extract_images(soup, url) + + # Extract links if requested + links = [] + if not config or config.extract_links: + links = self._extract_links(soup, url) + + return { + "title": extracted.get('title', metadata.title), + "text": extracted['text'], + "images": images, + "links": links, + "metadata": metadata + } + + def _sync_extract_main_content(self, soup: BeautifulSoup, url: str) -> Dict[str, str]: """ Extract main content using multiple strategies. Returns dict with 'title' and 'text' keys. """ - # Strategy 1: Try Readability algorithm (if available) - # Uncomment if readability-lxml is installed - # try: - # doc = Readability(str(soup)) - # summary = doc.summary() - # summary_soup = BeautifulSoup(summary, 'lxml') - # - # return { - # 'title': doc.title() or self._extract_title(soup), - # 'text': sanitize_text(summary_soup.get_text()) - # } - # except: - # pass - # Strategy 2: Look for common content containers content_selectors = [ 'main', @@ -579,8 +639,12 @@ async def _extract_main_content(self, soup: BeautifulSoup, url: str) -> Dict[str 'title': self._extract_title(soup), 'text': sanitize_text(soup.get_text()) } - - async def _extract_with_selectors( + + async def _extract_main_content(self, soup: BeautifulSoup, url: str) -> Dict[str, str]: + loop = asyncio.get_running_loop() + return await loop.run_in_executor(_scraping_cpu_executor, self._sync_extract_main_content, soup, url) + + def _sync_extract_with_selectors( self, soup: BeautifulSoup, selectors: Dict[str, str] @@ -623,28 +687,16 @@ async def _extract_with_selectors( result['text'] = '\n'.join(text_parts) return result - - def _extract_title(self, soup: BeautifulSoup) -> str: - """Extract page title using multiple strategies.""" - # Try standard title tag - title_tag = soup.find('title') - if title_tag: - return sanitize_text(title_tag.get_text()) - - # Try meta property - meta_title = soup.find('meta', property='og:title') - if meta_title and meta_title.get('content'): - return sanitize_text(meta_title['content']) - - # Try h1 tag - h1_tag = soup.find('h1') - if h1_tag: - return sanitize_text(h1_tag.get_text()) - - return "Untitled" - - async def extract_metadata(self, soup: BeautifulSoup, url: str) -> ContentMetadata: - """Extract structured metadata from the page.""" + + async def _extract_with_selectors( + self, + soup: BeautifulSoup, + selectors: Dict[str, str] + ) -> Dict[str, str]: + loop = asyncio.get_running_loop() + return await loop.run_in_executor(_scraping_cpu_executor, self._sync_extract_with_selectors, soup, selectors) + + def _sync_extract_metadata(self, soup: BeautifulSoup, url: str) -> ContentMetadata: metadata = ContentMetadata() # Extract title @@ -699,6 +751,30 @@ async def extract_metadata(self, soup: BeautifulSoup, url: str) -> ContentMetada pass return metadata + + async def extract_metadata(self, soup: BeautifulSoup, url: str) -> ContentMetadata: + """Extract structured metadata from the page.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor(_scraping_cpu_executor, self._sync_extract_metadata, soup, url) + + def _extract_title(self, soup: BeautifulSoup) -> str: + """Extract page title using multiple strategies.""" + # Try standard title tag + title_tag = soup.find('title') + if title_tag: + return sanitize_text(title_tag.get_text()) + + # Try meta property + meta_title = soup.find('meta', property='og:title') + if meta_title and meta_title.get('content'): + return sanitize_text(meta_title['content']) + + # Try h1 tag + h1_tag = soup.find('h1') + if h1_tag: + return sanitize_text(h1_tag.get_text()) + + return "Untitled" def _extract_images(self, soup: BeautifulSoup, base_url: str) -> List[str]: """Extract all images from the page.""" diff --git a/backend/tests/unit/test_non_blocking_scraper.py b/backend/tests/unit/test_non_blocking_scraper.py new file mode 100644 index 0000000..18c4908 --- /dev/null +++ b/backend/tests/unit/test_non_blocking_scraper.py @@ -0,0 +1,186 @@ +""" +Unit tests to verify BeautifulSoup parsing is offloaded to a thread pool and does not block the FastAPI event loop. +""" +import asyncio +import time +import pytest +from unittest.mock import Mock, AsyncMock, patch + +# Mock sent_tokenize to avoid requiring download of punkt_tab in tests +def mock_sent_tokenize(text): + return [s.strip() for s in text.split('.') if s.strip()] + +patch('app.utils.text_processing.sent_tokenize', side_effect=mock_sent_tokenize).start() + +from app.services.scraping import ContentScrapingService +from app.models.requests import ScrapingConfig +from bs4 import BeautifulSoup +from app.utils.text_processing import sanitize_text + +# A complex sample HTML for checking consistency +SAMPLE_HTML = """ + +
+This is the first paragraph of the main article content.
+This is the second paragraph of the main article content.
+Some text content for parsing.
\n" * 500 + huge_html = f"