diff --git a/napi/README.md b/napi/README.md index bbaacb3b..9d6f1a00 100644 --- a/napi/README.md +++ b/napi/README.md @@ -94,7 +94,7 @@ console.log(result.confidence) // 0.875 Extract text within bounding-box regions from a PDF. Designed for hybrid OCR pipelines where a layout model detects regions in rendered page images, and this function extracts text from the PDF structure for text-based pages — skipping GPU OCR. -Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues). When the cause is a suspected garbled text layer, `ocrReason` is set to `"suspected_garbled_text"`. +Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues). Known region causes are exposed in `ocrReason`; for example, a suspected garbled text layer reports `"suspected_garbled_text"`. A page skipped by the content-stream safety budget reports `"content_operation_limit"` in the page-level `ocrReasonsByPage` result. ```typescript import { extractTextInRegions } from '@firecrawl/pdf-inspector' @@ -157,7 +157,7 @@ interface PageRegionTexts { interface RegionText { text: string needsOcr: boolean // true when text is unreliable - ocrReason?: string // "suspected_garbled_text" when known + ocrReason?: string // machine-readable cause when known } interface OcrPdfResult { diff --git a/src/detector.rs b/src/detector.rs index 81aac146..a8e77806 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -62,6 +62,7 @@ pub struct PdfTypeResult { pub pages_needing_ocr: Vec, /// Per-page explanation for `pages_needing_ocr`: 1-indexed page → reason /// codes (`scanned`, `no_text`, `vector_text`, `suspected_garbled_text`). + /// The extraction stage can additionally report `content_operation_limit`. /// Only contains pages that need OCR. pub ocr_reasons_by_page: std::collections::BTreeMap>, } diff --git a/src/extractor/content_decode.rs b/src/extractor/content_decode.rs index aef346e3..4406fde8 100644 --- a/src/extractor/content_decode.rs +++ b/src/extractor/content_decode.rs @@ -9,8 +9,12 @@ use crate::PdfError; use lopdf::content::Content; /// Maximum content-stream operators decoded for a page or a single Form -/// XObject. Matches the previous post-decode skip threshold. -pub(crate) const MAX_PAGE_OPERATIONS: usize = 1_000_000; +/// XObject. +/// +/// Dense but valid CAD/Revit pages have been observed with up to 1.31 million +/// operators. Two million retains a hard allocation ceiling for pathological +/// streams while leaving headroom for that document family. +pub(crate) const MAX_PAGE_OPERATIONS: usize = 2_000_000; /// Decode `data` unless it contains more than `max_operations` operators. /// @@ -28,7 +32,7 @@ pub(crate) fn decode_content_bounded( .map_err(|e| PdfError::Parse(e.to_string())) } -fn content_exceeds_operation_limit(data: &[u8], max_operations: usize) -> bool { +pub(crate) fn content_exceeds_operation_limit(data: &[u8], max_operations: usize) -> bool { count_content_operators(data, max_operations.saturating_add(1)) > max_operations } diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 40c439f0..3bfba891 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -137,7 +137,8 @@ fn rise_adjusted(tm: &[f32; 6], rise: f32) -> [f32; 6] { ] } -/// Returns `(page_extraction, has_gid_fonts, coords_rotated, skipped_invisible)` +/// Returns `(page_extraction, has_gid_fonts, coords_rotated, skipped_invisible, +/// skipped_operation_limit)` /// where `has_gid_fonts` indicates the page uses fonts with unresolvable /// gid-encoded glyphs and `skipped_invisible` reports that invisible (Tr 3) /// text was present but suppressed — callers can use it to decide whether an @@ -150,7 +151,30 @@ pub(crate) fn extract_page_text_items( include_invisible: bool, style_cache: &mut FontStyleCache, form_budget: &mut FormWalkBudget, -) -> Result<(PageExtraction, bool, bool, bool), PdfError> { +) -> Result<(PageExtraction, bool, bool, bool, bool), PdfError> { + extract_page_text_items_with_operation_limit( + doc, + page_id, + page_num, + font_cmaps, + include_invisible, + style_cache, + form_budget, + super::content_decode::MAX_PAGE_OPERATIONS, + ) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn extract_page_text_items_with_operation_limit( + doc: &Document, + page_id: ObjectId, + page_num: u32, + font_cmaps: &FontCMaps, + include_invisible: bool, + style_cache: &mut FontStyleCache, + form_budget: &mut FormWalkBudget, + max_operations: usize, +) -> Result<(PageExtraction, bool, bool, bool, bool), PdfError> { let mut items = Vec::new(); let mut rects: Vec = Vec::new(); let mut clip_rects: Vec = Vec::new(); @@ -254,20 +278,24 @@ pub(crate) fn extract_page_text_items( // Content::decode parser, causing it to skip operators like ET and Q. let content_data = strip_pdf_comments(&content_data); - let content = match super::content_decode::decode_content_bounded( - &content_data, - super::content_decode::MAX_PAGE_OPERATIONS, - )? { - Some(content) => content, - None => { - log::warn!( - "page {}: skipping extraction — content stream exceeds {} operations", - page_num, - super::content_decode::MAX_PAGE_OPERATIONS - ); - return Ok(((Vec::new(), Vec::new(), Vec::new()), false, false, false)); - } - }; + let content = + match super::content_decode::decode_content_bounded(&content_data, max_operations)? { + Some(content) => content, + None => { + log::warn!( + "page {}: skipping extraction — content stream exceeds {} operations", + page_num, + max_operations + ); + return Ok(( + (Vec::new(), Vec::new(), Vec::new()), + false, + false, + false, + true, + )); + } + }; // Graphics state tracking let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix @@ -1363,6 +1391,7 @@ pub(crate) fn extract_page_text_items( has_gid_fonts, coords_rotated, skipped_invisible, + false, )) } @@ -1561,7 +1590,7 @@ mod tests { let (doc, page_id) = simple_doc_with_content(content); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _, _) = extract_page_text_items( + let ((items, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1574,6 +1603,28 @@ mod tests { items } + fn extract_simple_items_with_operation_limit( + content: &[u8], + max_operations: usize, + ) -> Vec { + use crate::tounicode::FontCMaps; + + let (doc, page_id) = simple_doc_with_content(content); + let font_cmaps = FontCMaps::from_doc(&doc); + let ((items, _, _), _, _, _, _) = extract_page_text_items_with_operation_limit( + &doc, + page_id, + 1, + &font_cmaps, + false, + &mut FontStyleCache::new(), + &mut FormWalkBudget::new(), + max_operations, + ) + .unwrap(); + items + } + #[test] fn test_dedup_rects_identical() { let mut rects = vec![rect(0.0, 0.0, 612.0, 792.0, 1); 3759]; @@ -1764,8 +1815,8 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET let mut doc = lopdf::Document::new(); - // "0 0 m\n" = 6 bytes per op, 1_100_000 ops → ~6.6 MB content stream - let ops_bytes = "0 0 m\n".repeat(1_100_000).into_bytes(); + // "0 0 m\n" = 6 bytes per op; stay above the current CAD-capable bound. + let ops_bytes = "0 0 m\n".repeat(2_100_000).into_bytes(); let stream = Stream::new(dictionary! {}, ops_bytes); let content_id = doc.add_object(Object::Stream(stream)); @@ -1801,7 +1852,7 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET &mut FormWalkBudget::new(), ) .unwrap(); - let ((items, rects, lines), _has_gid, _coords_rotated, _skipped_invisible) = result; + let ((items, rects, lines), _has_gid, _coords_rotated, _skipped_invisible, _) = result; assert!(items.is_empty()); assert!(rects.is_empty()); assert!(lines.is_empty()); @@ -1882,7 +1933,7 @@ BT 30 700 Tm <41> Tj ET"; doc.trailer.set("Root", Object::Reference(catalog_id)); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _, _) = extract_page_text_items( + let ((items, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1968,4 +2019,37 @@ BT 30 700 Tm <41> Tj ET"; "pages over the operator cap must not be decoded" ); } + + #[test] + fn reported_cad_operation_count_preserves_text() { + // #307: real CAD pages were observed at 1.07M and 1.31M operators. + // Check the reported size against the production cap without asking + // the regression test to allocate and walk a 1.1M-operation vector. + let mut content = Vec::with_capacity(1_100_002); + for _ in 0..1_100_000 { + content.extend_from_slice(b"q\n"); + } + content.extend_from_slice(b"BT /F1 12 Tf 72 720 Td (Hello CAD) Tj ET\n"); + assert!( + !super::super::content_decode::content_exceeds_operation_limit( + &content, + super::super::content_decode::MAX_PAGE_OPERATIONS, + ) + ); + + // Drive the same extraction path with a small cap: text below the cap + // survives, while the cap itself still fail-closes above it. + let mut small = b"BT /F1 12 Tf 72 720 Td (Hello CAD) Tj ET\n".to_vec(); + small.extend_from_slice(&b"q\n".repeat(10)); + let items = extract_simple_items_with_operation_limit(&small, 20); + let text = items + .iter() + .map(|item| item.text.as_str()) + .collect::(); + assert_eq!(text, "Hello CAD"); + + let mut over_limit = small.clone(); + over_limit.extend_from_slice(&b"q\n".repeat(20)); + assert!(extract_simple_items_with_operation_limit(&over_limit, 20).is_empty()); + } } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index c2c7b00b..51c2d70a 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -115,7 +115,7 @@ pub(crate) fn extract_text_with_positions_and_rects_with_password crate::validate_pdf_file(&path)?; let (doc, _) = crate::load_document_from_path_with_password(&path, password)?; let font_cmaps = FontCMaps::from_doc(&doc); - let (extraction, _thresholds, _gid_pages) = + let (extraction, _thresholds, _gid_pages, _operation_limit_pages) = extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; Ok(extraction) } @@ -142,7 +142,7 @@ pub(crate) fn extract_text_with_positions_mem_and_rects( crate::validate_pdf_bytes(buffer)?; let (doc, _) = crate::load_document_from_mem(buffer)?; let font_cmaps = FontCMaps::from_doc(&doc); - let (extraction, _thresholds, _gid_pages) = + let (extraction, _thresholds, _gid_pages, _operation_limit_pages) = extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; Ok(extraction) } @@ -161,7 +161,7 @@ pub(crate) fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { extract_positioned_text_impl(doc, font_cmaps, page_filter, false, None) } @@ -172,7 +172,7 @@ pub(crate) fn extract_positioned_text_with_folio_context( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { extract_positioned_text_with_folio_context_impl(doc, font_cmaps, page_filter, false) } @@ -181,7 +181,7 @@ pub(crate) fn extract_positioned_text_include_invisible_with_folio_context( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { extract_positioned_text_with_folio_context_impl(doc, font_cmaps, page_filter, true) } @@ -190,7 +190,7 @@ fn extract_positioned_text_with_folio_context_impl( font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, include_invisible: bool, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { let Some(required_pages) = page_filter else { return extract_positioned_text_impl(doc, font_cmaps, None, include_invisible, None); }; @@ -199,6 +199,7 @@ fn extract_positioned_text_with_folio_context_impl( (mut selected_items, mut selected_rects, mut selected_lines), mut page_thresholds, mut gid_encoded_pages, + mut operation_limit_pages, ) = extract_positioned_text_impl( doc, font_cmaps, @@ -211,6 +212,7 @@ fn extract_positioned_text_with_folio_context_impl( (selected_items, selected_rects, selected_lines), page_thresholds, gid_encoded_pages, + operation_limit_pages, )); } @@ -220,23 +222,29 @@ fn extract_positioned_text_with_folio_context_impl( .copied() .filter(|page| !required_pages.contains(page)) .collect(); - let ((context_items, context_rects, context_lines), context_thresholds, context_gid_pages) = - extract_positioned_text_impl( - doc, - font_cmaps, - Some(&context_pages), - include_invisible, - Some(required_pages), - )?; + let ( + (context_items, context_rects, context_lines), + context_thresholds, + context_gid_pages, + context_operation_limit_pages, + ) = extract_positioned_text_impl( + doc, + font_cmaps, + Some(&context_pages), + include_invisible, + Some(required_pages), + )?; selected_items.extend(context_items); selected_rects.extend(context_rects); selected_lines.extend(context_lines); page_thresholds.extend(context_thresholds); gid_encoded_pages.extend(context_gid_pages); + operation_limit_pages.extend(context_operation_limit_pages); Ok(( (selected_items, selected_rects, selected_lines), page_thresholds, gid_encoded_pages, + operation_limit_pages, )) } @@ -246,7 +254,7 @@ pub(crate) fn extract_positioned_text_for_document_analysis( doc: &Document, font_cmaps: &FontCMaps, required_pages: &HashSet, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { extract_positioned_text_impl(doc, font_cmaps, None, false, Some(required_pages)) } @@ -256,13 +264,14 @@ fn extract_positioned_text_impl( page_filter: Option<&HashSet>, include_invisible: bool, required_pages: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { let pages = doc.get_pages(); let mut all_items = Vec::new(); let mut all_rects = Vec::new(); let mut all_lines = Vec::new(); let mut page_thresholds: PageThresholds = HashMap::new(); let mut gid_encoded_pages: HashSet = HashSet::new(); + let mut operation_limit_pages: HashSet = HashSet::new(); // Embedded-font style flags are document-scoped: the same font program // is shared across pages, so parse it once, not once per page. let mut style_cache = FontStyleCache::new(); @@ -286,20 +295,26 @@ fn extract_positioned_text_impl( &mut style_cache, &mut FormWalkBudget::new(), ); - let ((mut items, mut rects, mut lines), has_gid_fonts, coords_rotated, _skipped_invisible) = - match page_result { - Ok(extraction) => extraction, - Err(error) - if required_pages.is_some_and(|required| !required.contains(page_num)) => - { - debug!( - "page {}: skipping context-only extraction error: {}", - page_num, error - ); - continue; - } - Err(error) => return Err(error), - }; + let ( + (mut items, mut rects, mut lines), + has_gid_fonts, + coords_rotated, + _skipped_invisible, + skipped_operation_limit, + ) = match page_result { + Ok(extraction) => extraction, + Err(error) if required_pages.is_some_and(|required| !required.contains(page_num)) => { + debug!( + "page {}: skipping context-only extraction error: {}", + page_num, error + ); + continue; + } + Err(error) => return Err(error), + }; + if skipped_operation_limit { + operation_limit_pages.insert(*page_num); + } // Clip to the visible page box: single-page extracts and imposed // spreads keep neighboring pages' content in the stream, positioned // outside the CropBox. Extracting it interleaves invisible text into @@ -438,6 +453,7 @@ fn extract_positioned_text_impl( (all_items, all_rects, all_lines), page_thresholds, gid_encoded_pages, + operation_limit_pages, )) } diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index 66b3d7e2..beefd099 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -1032,7 +1032,7 @@ mod tests { let (doc, page_id) = page_invoking_form(doc, root); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _, _) = extract_page_text_items( + let ((items, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1062,7 +1062,7 @@ mod tests { let font_cmaps = FontCMaps::from_doc(&doc); // Root + leaf = 2 invocations on the first pass. let mut budget = FormWalkBudget::with_limits(2, MAX_FORM_XOBJECT_OPERATIONS); - let ((first, _, _), _, _, _) = extract_page_text_items( + let ((first, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1075,7 +1075,7 @@ mod tests { assert_eq!(first.iter().filter(|item| item.text == "X").count(), 1); assert!(!budget.was_truncated()); - let ((second, _, _), _, _, _) = extract_page_text_items( + let ((second, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1145,7 +1145,7 @@ mod tests { fn form_items(form_content: &[u8]) -> Vec { let (doc, page_id) = doc_with_form_content(form_content); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _, _) = extract_page_text_items( + let ((items, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, diff --git a/src/lib.rs b/src/lib.rs index 2b0ecce2..f53ec3d1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -119,6 +119,13 @@ pub const OCR_REASON_NO_TEXT: &str = "no_text"; /// rather than real text operators, so it cannot be extracted as characters. pub const OCR_REASON_VECTOR_TEXT: &str = "vector_text"; +/// OCR reason: the page exceeded the bounded content-stream operator budget. +/// +/// Extraction is skipped rather than decoding an unbounded operation vector. +/// The reason is reported separately from `no_text` so callers can distinguish +/// a safety limit from a genuinely empty page. +pub const OCR_REASON_CONTENT_OPERATION_LIMIT: &str = "content_operation_limit"; + // ========================================================================= // Result type // ========================================================================= @@ -508,7 +515,7 @@ fn extract_pages_markdown_mem_impl( .filter_map(|page| page.checked_add(1)) .collect() }); - let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) = + let ((all_items, all_rects, all_lines), page_thresholds, gid_pages, operation_limit_pages) = if let Some(required_pages) = required_pages.as_ref() { extractor::extract_positioned_text_for_document_analysis( &doc, @@ -594,6 +601,13 @@ fn extract_pages_markdown_mem_impl( let has_gid = gid_pages.contains(&page_1idx); let has_text_quality_issue = text_quality.pages_needing_ocr.contains(&page_1idx); + if operation_limit_pages.contains(&page_1idx) { + add_ocr_reason( + &mut ocr_reasons_by_page, + page_1idx, + OCR_REASON_CONTENT_OPERATION_LIMIT, + ); + } // A page can extract cleanly (no decoding issues, non-empty text) // while still being fundamentally a scan: a full-page raster with @@ -998,7 +1012,7 @@ pub fn extract_text_in_regions_mem( // with the invisible-layer retry below so one page cannot consume two // full expansion budgets. let mut form_budget = extractor::FormWalkBudget::new(); - let ((mut items, _rects, _lines), mut has_gid, mut coords_rotated, skipped_invisible) = + let ((mut items, _rects, _lines), mut has_gid, mut coords_rotated, skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -1027,7 +1041,7 @@ pub fn extract_text_in_regions_mem( !matches!(it.item_type, types::ItemType::Image) && !it.text.trim().is_empty() }); if skipped_invisible && !has_visible_text { - if let Ok(((inv_items, _inv_rects, _inv_lines), inv_gid, inv_rotated, _)) = + if let Ok(((inv_items, _inv_rects, _inv_lines), inv_gid, inv_rotated, _, _)) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -1210,7 +1224,7 @@ pub fn extract_tables_in_regions_mem( let height = get_page_height(&doc, page_id).unwrap_or(792.0); page_heights.insert(*page_num, height); - let ((mut items, rects, lines), has_gid, coords_rotated, _skipped_invisible) = + let ((mut items, rects, lines), has_gid, coords_rotated, _skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -1522,7 +1536,7 @@ pub fn detect_vector_grid_in_region_mem( let needed_pages = HashSet::from([page_1idx]); let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages)); let page_h = get_page_height(&doc, page_id).unwrap_or(792.0); - let ((mut items, rects, lines), _has_gid, coords_rotated, _skipped_invisible) = + let ((mut items, rects, lines), _has_gid, coords_rotated, _skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -1717,7 +1731,7 @@ mod vector_grid_tests { let &page_id = pages.get(&1).unwrap(); let needed: HashSet = HashSet::from([1]); let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed)); - let ((items, rects, _lines), _has_gid, _rotated, _skipped_invisible) = + let ((items, rects, _lines), _has_gid, _rotated, _skipped_invisible, _) = extract_page_text_items( &doc, page_id, @@ -1761,7 +1775,7 @@ mod vector_grid_tests { let &page_id = pages.get(&page_num).unwrap(); let needed: HashSet = HashSet::from([page_num]); let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed)); - let ((items, rects, _lines), _has_gid, _rotated, _skipped_invisible) = + let ((items, rects, _lines), _has_gid, _rotated, _skipped_invisible, _) = extract_page_text_items( &doc, page_id, @@ -2499,7 +2513,7 @@ pub fn extract_tables_with_structure_cells_mem( let height = get_page_height(&doc, page_id).unwrap_or(792.0); page_heights.insert(*page_num, height); - let ((mut items, _rects, _lines), _has_gid, coords_rotated, _skipped_invisible) = + let ((mut items, _rects, _lines), _has_gid, coords_rotated, _skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -3302,7 +3316,7 @@ fn detect_tsr_quality_issue( let mut needed: HashSet = HashSet::new(); needed.insert(page_1idx); let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed)); - let ((mut items, _rects, _lines), _has_gid, coords_rotated, _skipped_invisible) = + let ((mut items, _rects, _lines), _has_gid, coords_rotated, _skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -4109,7 +4123,7 @@ fn process_document( // (mostly non-alphanumeric), retry with invisible (Tr=3) text included. // This unlocks OCR text layers behind scanned images. if pdf_type == PdfType::Mixed { - if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _)| e) { + if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _, _)| e) { let sample: String = items .iter() .filter(|item| { @@ -4177,8 +4191,26 @@ fn process_document( text_quality_pages, text_quality_reasons_by_page, ) = match extracted { - Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => { + Some(( + (items, rects, lines), + page_thresholds, + gid_encoded_pages, + operation_limit_pages, + )) => { let mut ocr_reasons_by_page = BTreeMap::new(); + for page in operation_limit_pages { + if options + .page_filter + .as_ref() + .is_none_or(|filter| filter.contains(&page)) + { + add_ocr_reason( + &mut ocr_reasons_by_page, + page, + OCR_REASON_CONTENT_OPERATION_LIMIT, + ); + } + } // For TextBased PDFs with pages flagged for OCR (Identity-H or // Type3 fonts without ToUnicode), check whether the CID-as-Unicode @@ -4302,8 +4334,11 @@ fn process_document( )) }; - let enc = !ocr_reasons_by_page.is_empty() - || text_quality.has_encoding_issues + let enc = ocr_reasons_by_page.values().any(|reasons| { + reasons + .iter() + .any(|reason| reason != OCR_REASON_CONTENT_OPERATION_LIMIT) + }) || text_quality.has_encoding_issues || md.as_ref().is_some_and(|m| detect_encoding_issues(m)); ( md, @@ -4375,6 +4410,22 @@ fn process_document( } pages_needing_ocr.sort_unstable(); } + let operation_limit_pages: Vec<_> = text_quality_reasons_by_page + .iter() + .filter(|(_, reasons)| { + reasons + .iter() + .any(|reason| reason == OCR_REASON_CONTENT_OPERATION_LIMIT) + }) + .map(|(page, _)| *page) + .collect(); + for page in operation_limit_pages { + if !pages_needing_ocr.contains(&page) { + pages_needing_ocr.push(page); + } + } + pages_needing_ocr.sort_unstable(); + pages_needing_ocr.dedup(); // Detect sparse extraction: when a TEXT-BASED PDF produces very few // characters per page, the text is likely embedded in images/forms @@ -4414,8 +4465,9 @@ fn process_document( processing_time_ms: start.elapsed_ms(), pages_needing_ocr, ocr_reasons_by_page: { - // Detector reasons (scanned / no_text / vector_text / garbled) merged - // with the markdown-stage garbled detection, deduped per page. + // Detector reasons (scanned / no_text / vector_text / garbled) + // merged with extraction-stage operation-limit and garbled reasons, + // deduped per page. let mut merged = detection_ocr_reasons; merge_ocr_reasons(&mut merged, text_quality_reasons_by_page); page_ocr_reasons_vec(merged) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 1491330b..8389c5ba 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1968,6 +1968,56 @@ fn test_extract_pages_mem_shifted_cipher_tounicode_needs_ocr() { ); } +#[test] +fn test_extract_pages_mem_reports_content_operation_limit() { + let mut content = Vec::with_capacity(4_200_002); + for _ in 0..2_100_000 { + content.extend_from_slice(b"q\n"); + } + content.extend_from_slice(b"BT /F1 12 Tf 72 720 Td (Hello CAD) Tj ET\n"); + let buf = make_text_pdf( + std::str::from_utf8(&content).expect("test content is ASCII"), + "0 0 612 792", + ); + + let result = extract_pages_markdown_mem(&buf, None).unwrap(); + assert_eq!(result.pages.len(), 1); + assert!(result.pages[0].needs_ocr); + assert!(result.pages[0].markdown.is_empty()); + assert_eq!(result.pages_needing_ocr, vec![1]); + assert_eq!( + result.pages[0].ocr_reason.as_deref(), + Some(pdf_inspector::OCR_REASON_CONTENT_OPERATION_LIMIT) + ); +} + +#[test] +fn test_process_pdf_mem_reports_operation_limit_without_encoding_issue() { + let mut content = Vec::with_capacity(4_200_002); + for _ in 0..2_100_000 { + content.extend_from_slice(b"q\n"); + } + content.extend_from_slice(b"BT /F1 12 Tf 72 720 Td (Hello CAD) Tj ET\n"); + let buf = make_text_pdf( + std::str::from_utf8(&content).expect("test content is ASCII"), + "0 0 612 792", + ); + + let result = process_pdf_mem(&buf).unwrap(); + + assert_eq!(result.page_count, 1); + assert_eq!(result.pages_needing_ocr, vec![1]); + assert!(!result.has_encoding_issues); + assert_eq!( + result + .ocr_reasons_by_page + .iter() + .flat_map(|reason| reason.reasons.iter().cloned()) + .collect::>(), + vec![pdf_inspector::OCR_REASON_CONTENT_OPERATION_LIMIT.to_string()] + ); +} + #[test] fn test_extract_regions_mem_multiple_regions_per_page() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();