From ba95395d2d6273d3ad55a243e8d0a2666fd4d91d Mon Sep 17 00:00:00 2001 From: yzxcj797 <1784931579@qq.com> Date: Tue, 18 Aug 2026 17:40:32 +0800 Subject: [PATCH 1/2] Support dense CAD content streams Raise the bounded page operator budget to two million so valid CAD pages with roughly 1.3 million operations retain their text. Pages above the safety budget now report a dedicated content_operation_limit OCR reason instead of looking like an empty text page. --- napi/README.md | 4 +- src/detector.rs | 1 + src/extractor/content_decode.rs | 10 ++- src/extractor/content_stream.rs | 126 ++++++++++++++++++++++++++------ src/extractor/mod.rs | 76 +++++++++++-------- src/extractor/xobjects.rs | 8 +- src/lib.rs | 59 +++++++++++---- tests/integration_tests.rs | 23 ++++++ 8 files changed, 234 insertions(+), 73 deletions(-) diff --git a/napi/README.md b/napi/README.md index bbaacb3b..6a8ca91b 100644 --- a/napi/README.md +++ b/napi/README.md @@ -94,7 +94,7 @@ console.log(result.confidence) // 0.875 Extract text within bounding-box regions from a PDF. Designed for hybrid OCR pipelines where a layout model detects regions in rendered page images, and this function extracts text from the PDF structure for text-based pages — skipping GPU OCR. -Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues). When the cause is a suspected garbled text layer, `ocrReason` is set to `"suspected_garbled_text"`. +Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues). Known causes are exposed in `ocrReason`; for example, a suspected garbled text layer reports `"suspected_garbled_text"`, and a page skipped by the content-stream safety budget reports `"content_operation_limit"`. ```typescript import { extractTextInRegions } from '@firecrawl/pdf-inspector' @@ -157,7 +157,7 @@ interface PageRegionTexts { interface RegionText { text: string needsOcr: boolean // true when text is unreliable - ocrReason?: string // "suspected_garbled_text" when known + ocrReason?: string // machine-readable cause when known } interface OcrPdfResult { diff --git a/src/detector.rs b/src/detector.rs index 81aac146..a8e77806 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -62,6 +62,7 @@ pub struct PdfTypeResult { pub pages_needing_ocr: Vec, /// Per-page explanation for `pages_needing_ocr`: 1-indexed page → reason /// codes (`scanned`, `no_text`, `vector_text`, `suspected_garbled_text`). + /// The extraction stage can additionally report `content_operation_limit`. /// Only contains pages that need OCR. pub ocr_reasons_by_page: std::collections::BTreeMap>, } diff --git a/src/extractor/content_decode.rs b/src/extractor/content_decode.rs index aef346e3..4406fde8 100644 --- a/src/extractor/content_decode.rs +++ b/src/extractor/content_decode.rs @@ -9,8 +9,12 @@ use crate::PdfError; use lopdf::content::Content; /// Maximum content-stream operators decoded for a page or a single Form -/// XObject. Matches the previous post-decode skip threshold. -pub(crate) const MAX_PAGE_OPERATIONS: usize = 1_000_000; +/// XObject. +/// +/// Dense but valid CAD/Revit pages have been observed with up to 1.31 million +/// operators. Two million retains a hard allocation ceiling for pathological +/// streams while leaving headroom for that document family. +pub(crate) const MAX_PAGE_OPERATIONS: usize = 2_000_000; /// Decode `data` unless it contains more than `max_operations` operators. /// @@ -28,7 +32,7 @@ pub(crate) fn decode_content_bounded( .map_err(|e| PdfError::Parse(e.to_string())) } -fn content_exceeds_operation_limit(data: &[u8], max_operations: usize) -> bool { +pub(crate) fn content_exceeds_operation_limit(data: &[u8], max_operations: usize) -> bool { count_content_operators(data, max_operations.saturating_add(1)) > max_operations } diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index 40c439f0..3bfba891 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -137,7 +137,8 @@ fn rise_adjusted(tm: &[f32; 6], rise: f32) -> [f32; 6] { ] } -/// Returns `(page_extraction, has_gid_fonts, coords_rotated, skipped_invisible)` +/// Returns `(page_extraction, has_gid_fonts, coords_rotated, skipped_invisible, +/// skipped_operation_limit)` /// where `has_gid_fonts` indicates the page uses fonts with unresolvable /// gid-encoded glyphs and `skipped_invisible` reports that invisible (Tr 3) /// text was present but suppressed — callers can use it to decide whether an @@ -150,7 +151,30 @@ pub(crate) fn extract_page_text_items( include_invisible: bool, style_cache: &mut FontStyleCache, form_budget: &mut FormWalkBudget, -) -> Result<(PageExtraction, bool, bool, bool), PdfError> { +) -> Result<(PageExtraction, bool, bool, bool, bool), PdfError> { + extract_page_text_items_with_operation_limit( + doc, + page_id, + page_num, + font_cmaps, + include_invisible, + style_cache, + form_budget, + super::content_decode::MAX_PAGE_OPERATIONS, + ) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn extract_page_text_items_with_operation_limit( + doc: &Document, + page_id: ObjectId, + page_num: u32, + font_cmaps: &FontCMaps, + include_invisible: bool, + style_cache: &mut FontStyleCache, + form_budget: &mut FormWalkBudget, + max_operations: usize, +) -> Result<(PageExtraction, bool, bool, bool, bool), PdfError> { let mut items = Vec::new(); let mut rects: Vec = Vec::new(); let mut clip_rects: Vec = Vec::new(); @@ -254,20 +278,24 @@ pub(crate) fn extract_page_text_items( // Content::decode parser, causing it to skip operators like ET and Q. let content_data = strip_pdf_comments(&content_data); - let content = match super::content_decode::decode_content_bounded( - &content_data, - super::content_decode::MAX_PAGE_OPERATIONS, - )? { - Some(content) => content, - None => { - log::warn!( - "page {}: skipping extraction — content stream exceeds {} operations", - page_num, - super::content_decode::MAX_PAGE_OPERATIONS - ); - return Ok(((Vec::new(), Vec::new(), Vec::new()), false, false, false)); - } - }; + let content = + match super::content_decode::decode_content_bounded(&content_data, max_operations)? { + Some(content) => content, + None => { + log::warn!( + "page {}: skipping extraction — content stream exceeds {} operations", + page_num, + max_operations + ); + return Ok(( + (Vec::new(), Vec::new(), Vec::new()), + false, + false, + false, + true, + )); + } + }; // Graphics state tracking let mut ctm = [1.0f32, 0.0, 0.0, 1.0, 0.0, 0.0]; // Current Transformation Matrix @@ -1363,6 +1391,7 @@ pub(crate) fn extract_page_text_items( has_gid_fonts, coords_rotated, skipped_invisible, + false, )) } @@ -1561,7 +1590,7 @@ mod tests { let (doc, page_id) = simple_doc_with_content(content); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _, _) = extract_page_text_items( + let ((items, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1574,6 +1603,28 @@ mod tests { items } + fn extract_simple_items_with_operation_limit( + content: &[u8], + max_operations: usize, + ) -> Vec { + use crate::tounicode::FontCMaps; + + let (doc, page_id) = simple_doc_with_content(content); + let font_cmaps = FontCMaps::from_doc(&doc); + let ((items, _, _), _, _, _, _) = extract_page_text_items_with_operation_limit( + &doc, + page_id, + 1, + &font_cmaps, + false, + &mut FontStyleCache::new(), + &mut FormWalkBudget::new(), + max_operations, + ) + .unwrap(); + items + } + #[test] fn test_dedup_rects_identical() { let mut rects = vec![rect(0.0, 0.0, 612.0, 792.0, 1); 3759]; @@ -1764,8 +1815,8 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET let mut doc = lopdf::Document::new(); - // "0 0 m\n" = 6 bytes per op, 1_100_000 ops → ~6.6 MB content stream - let ops_bytes = "0 0 m\n".repeat(1_100_000).into_bytes(); + // "0 0 m\n" = 6 bytes per op; stay above the current CAD-capable bound. + let ops_bytes = "0 0 m\n".repeat(2_100_000).into_bytes(); let stream = Stream::new(dictionary! {}, ops_bytes); let content_id = doc.add_object(Object::Stream(stream)); @@ -1801,7 +1852,7 @@ BT /F1 12 Tf 0 1 -1 0 240 100 Tm (WORLD) Tj ET &mut FormWalkBudget::new(), ) .unwrap(); - let ((items, rects, lines), _has_gid, _coords_rotated, _skipped_invisible) = result; + let ((items, rects, lines), _has_gid, _coords_rotated, _skipped_invisible, _) = result; assert!(items.is_empty()); assert!(rects.is_empty()); assert!(lines.is_empty()); @@ -1882,7 +1933,7 @@ BT 30 700 Tm <41> Tj ET"; doc.trailer.set("Root", Object::Reference(catalog_id)); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _, _) = extract_page_text_items( + let ((items, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1968,4 +2019,37 @@ BT 30 700 Tm <41> Tj ET"; "pages over the operator cap must not be decoded" ); } + + #[test] + fn reported_cad_operation_count_preserves_text() { + // #307: real CAD pages were observed at 1.07M and 1.31M operators. + // Check the reported size against the production cap without asking + // the regression test to allocate and walk a 1.1M-operation vector. + let mut content = Vec::with_capacity(1_100_002); + for _ in 0..1_100_000 { + content.extend_from_slice(b"q\n"); + } + content.extend_from_slice(b"BT /F1 12 Tf 72 720 Td (Hello CAD) Tj ET\n"); + assert!( + !super::super::content_decode::content_exceeds_operation_limit( + &content, + super::super::content_decode::MAX_PAGE_OPERATIONS, + ) + ); + + // Drive the same extraction path with a small cap: text below the cap + // survives, while the cap itself still fail-closes above it. + let mut small = b"BT /F1 12 Tf 72 720 Td (Hello CAD) Tj ET\n".to_vec(); + small.extend_from_slice(&b"q\n".repeat(10)); + let items = extract_simple_items_with_operation_limit(&small, 20); + let text = items + .iter() + .map(|item| item.text.as_str()) + .collect::(); + assert_eq!(text, "Hello CAD"); + + let mut over_limit = small.clone(); + over_limit.extend_from_slice(&b"q\n".repeat(20)); + assert!(extract_simple_items_with_operation_limit(&over_limit, 20).is_empty()); + } } diff --git a/src/extractor/mod.rs b/src/extractor/mod.rs index c2c7b00b..51c2d70a 100644 --- a/src/extractor/mod.rs +++ b/src/extractor/mod.rs @@ -115,7 +115,7 @@ pub(crate) fn extract_text_with_positions_and_rects_with_password crate::validate_pdf_file(&path)?; let (doc, _) = crate::load_document_from_path_with_password(&path, password)?; let font_cmaps = FontCMaps::from_doc(&doc); - let (extraction, _thresholds, _gid_pages) = + let (extraction, _thresholds, _gid_pages, _operation_limit_pages) = extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; Ok(extraction) } @@ -142,7 +142,7 @@ pub(crate) fn extract_text_with_positions_mem_and_rects( crate::validate_pdf_bytes(buffer)?; let (doc, _) = crate::load_document_from_mem(buffer)?; let font_cmaps = FontCMaps::from_doc(&doc); - let (extraction, _thresholds, _gid_pages) = + let (extraction, _thresholds, _gid_pages, _operation_limit_pages) = extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)?; Ok(extraction) } @@ -161,7 +161,7 @@ pub(crate) fn extract_positioned_text_from_doc( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { extract_positioned_text_impl(doc, font_cmaps, page_filter, false, None) } @@ -172,7 +172,7 @@ pub(crate) fn extract_positioned_text_with_folio_context( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { extract_positioned_text_with_folio_context_impl(doc, font_cmaps, page_filter, false) } @@ -181,7 +181,7 @@ pub(crate) fn extract_positioned_text_include_invisible_with_folio_context( doc: &Document, font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { extract_positioned_text_with_folio_context_impl(doc, font_cmaps, page_filter, true) } @@ -190,7 +190,7 @@ fn extract_positioned_text_with_folio_context_impl( font_cmaps: &FontCMaps, page_filter: Option<&HashSet>, include_invisible: bool, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { let Some(required_pages) = page_filter else { return extract_positioned_text_impl(doc, font_cmaps, None, include_invisible, None); }; @@ -199,6 +199,7 @@ fn extract_positioned_text_with_folio_context_impl( (mut selected_items, mut selected_rects, mut selected_lines), mut page_thresholds, mut gid_encoded_pages, + mut operation_limit_pages, ) = extract_positioned_text_impl( doc, font_cmaps, @@ -211,6 +212,7 @@ fn extract_positioned_text_with_folio_context_impl( (selected_items, selected_rects, selected_lines), page_thresholds, gid_encoded_pages, + operation_limit_pages, )); } @@ -220,23 +222,29 @@ fn extract_positioned_text_with_folio_context_impl( .copied() .filter(|page| !required_pages.contains(page)) .collect(); - let ((context_items, context_rects, context_lines), context_thresholds, context_gid_pages) = - extract_positioned_text_impl( - doc, - font_cmaps, - Some(&context_pages), - include_invisible, - Some(required_pages), - )?; + let ( + (context_items, context_rects, context_lines), + context_thresholds, + context_gid_pages, + context_operation_limit_pages, + ) = extract_positioned_text_impl( + doc, + font_cmaps, + Some(&context_pages), + include_invisible, + Some(required_pages), + )?; selected_items.extend(context_items); selected_rects.extend(context_rects); selected_lines.extend(context_lines); page_thresholds.extend(context_thresholds); gid_encoded_pages.extend(context_gid_pages); + operation_limit_pages.extend(context_operation_limit_pages); Ok(( (selected_items, selected_rects, selected_lines), page_thresholds, gid_encoded_pages, + operation_limit_pages, )) } @@ -246,7 +254,7 @@ pub(crate) fn extract_positioned_text_for_document_analysis( doc: &Document, font_cmaps: &FontCMaps, required_pages: &HashSet, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { extract_positioned_text_impl(doc, font_cmaps, None, false, Some(required_pages)) } @@ -256,13 +264,14 @@ fn extract_positioned_text_impl( page_filter: Option<&HashSet>, include_invisible: bool, required_pages: Option<&HashSet>, -) -> Result<(PageExtraction, PageThresholds, HashSet), PdfError> { +) -> Result<(PageExtraction, PageThresholds, HashSet, HashSet), PdfError> { let pages = doc.get_pages(); let mut all_items = Vec::new(); let mut all_rects = Vec::new(); let mut all_lines = Vec::new(); let mut page_thresholds: PageThresholds = HashMap::new(); let mut gid_encoded_pages: HashSet = HashSet::new(); + let mut operation_limit_pages: HashSet = HashSet::new(); // Embedded-font style flags are document-scoped: the same font program // is shared across pages, so parse it once, not once per page. let mut style_cache = FontStyleCache::new(); @@ -286,20 +295,26 @@ fn extract_positioned_text_impl( &mut style_cache, &mut FormWalkBudget::new(), ); - let ((mut items, mut rects, mut lines), has_gid_fonts, coords_rotated, _skipped_invisible) = - match page_result { - Ok(extraction) => extraction, - Err(error) - if required_pages.is_some_and(|required| !required.contains(page_num)) => - { - debug!( - "page {}: skipping context-only extraction error: {}", - page_num, error - ); - continue; - } - Err(error) => return Err(error), - }; + let ( + (mut items, mut rects, mut lines), + has_gid_fonts, + coords_rotated, + _skipped_invisible, + skipped_operation_limit, + ) = match page_result { + Ok(extraction) => extraction, + Err(error) if required_pages.is_some_and(|required| !required.contains(page_num)) => { + debug!( + "page {}: skipping context-only extraction error: {}", + page_num, error + ); + continue; + } + Err(error) => return Err(error), + }; + if skipped_operation_limit { + operation_limit_pages.insert(*page_num); + } // Clip to the visible page box: single-page extracts and imposed // spreads keep neighboring pages' content in the stream, positioned // outside the CropBox. Extracting it interleaves invisible text into @@ -438,6 +453,7 @@ fn extract_positioned_text_impl( (all_items, all_rects, all_lines), page_thresholds, gid_encoded_pages, + operation_limit_pages, )) } diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index 66b3d7e2..beefd099 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -1032,7 +1032,7 @@ mod tests { let (doc, page_id) = page_invoking_form(doc, root); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _, _) = extract_page_text_items( + let ((items, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1062,7 +1062,7 @@ mod tests { let font_cmaps = FontCMaps::from_doc(&doc); // Root + leaf = 2 invocations on the first pass. let mut budget = FormWalkBudget::with_limits(2, MAX_FORM_XOBJECT_OPERATIONS); - let ((first, _, _), _, _, _) = extract_page_text_items( + let ((first, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1075,7 +1075,7 @@ mod tests { assert_eq!(first.iter().filter(|item| item.text == "X").count(), 1); assert!(!budget.was_truncated()); - let ((second, _, _), _, _, _) = extract_page_text_items( + let ((second, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, @@ -1145,7 +1145,7 @@ mod tests { fn form_items(form_content: &[u8]) -> Vec { let (doc, page_id) = doc_with_form_content(form_content); let font_cmaps = FontCMaps::from_doc(&doc); - let ((items, _, _), _, _, _) = extract_page_text_items( + let ((items, _, _), _, _, _, _) = extract_page_text_items( &doc, page_id, 1, diff --git a/src/lib.rs b/src/lib.rs index 2b0ecce2..d10d5084 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -119,6 +119,13 @@ pub const OCR_REASON_NO_TEXT: &str = "no_text"; /// rather than real text operators, so it cannot be extracted as characters. pub const OCR_REASON_VECTOR_TEXT: &str = "vector_text"; +/// OCR reason: the page exceeded the bounded content-stream operator budget. +/// +/// Extraction is skipped rather than decoding an unbounded operation vector. +/// The reason is reported separately from `no_text` so callers can distinguish +/// a safety limit from a genuinely empty page. +pub const OCR_REASON_CONTENT_OPERATION_LIMIT: &str = "content_operation_limit"; + // ========================================================================= // Result type // ========================================================================= @@ -508,7 +515,7 @@ fn extract_pages_markdown_mem_impl( .filter_map(|page| page.checked_add(1)) .collect() }); - let ((all_items, all_rects, all_lines), page_thresholds, gid_pages) = + let ((all_items, all_rects, all_lines), page_thresholds, gid_pages, operation_limit_pages) = if let Some(required_pages) = required_pages.as_ref() { extractor::extract_positioned_text_for_document_analysis( &doc, @@ -594,6 +601,13 @@ fn extract_pages_markdown_mem_impl( let has_gid = gid_pages.contains(&page_1idx); let has_text_quality_issue = text_quality.pages_needing_ocr.contains(&page_1idx); + if operation_limit_pages.contains(&page_1idx) { + add_ocr_reason( + &mut ocr_reasons_by_page, + page_1idx, + OCR_REASON_CONTENT_OPERATION_LIMIT, + ); + } // A page can extract cleanly (no decoding issues, non-empty text) // while still being fundamentally a scan: a full-page raster with @@ -998,7 +1012,7 @@ pub fn extract_text_in_regions_mem( // with the invisible-layer retry below so one page cannot consume two // full expansion budgets. let mut form_budget = extractor::FormWalkBudget::new(); - let ((mut items, _rects, _lines), mut has_gid, mut coords_rotated, skipped_invisible) = + let ((mut items, _rects, _lines), mut has_gid, mut coords_rotated, skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -1027,7 +1041,7 @@ pub fn extract_text_in_regions_mem( !matches!(it.item_type, types::ItemType::Image) && !it.text.trim().is_empty() }); if skipped_invisible && !has_visible_text { - if let Ok(((inv_items, _inv_rects, _inv_lines), inv_gid, inv_rotated, _)) = + if let Ok(((inv_items, _inv_rects, _inv_lines), inv_gid, inv_rotated, _, _)) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -1210,7 +1224,7 @@ pub fn extract_tables_in_regions_mem( let height = get_page_height(&doc, page_id).unwrap_or(792.0); page_heights.insert(*page_num, height); - let ((mut items, rects, lines), has_gid, coords_rotated, _skipped_invisible) = + let ((mut items, rects, lines), has_gid, coords_rotated, _skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -1522,7 +1536,7 @@ pub fn detect_vector_grid_in_region_mem( let needed_pages = HashSet::from([page_1idx]); let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed_pages)); let page_h = get_page_height(&doc, page_id).unwrap_or(792.0); - let ((mut items, rects, lines), _has_gid, coords_rotated, _skipped_invisible) = + let ((mut items, rects, lines), _has_gid, coords_rotated, _skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -1717,7 +1731,7 @@ mod vector_grid_tests { let &page_id = pages.get(&1).unwrap(); let needed: HashSet = HashSet::from([1]); let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed)); - let ((items, rects, _lines), _has_gid, _rotated, _skipped_invisible) = + let ((items, rects, _lines), _has_gid, _rotated, _skipped_invisible, _) = extract_page_text_items( &doc, page_id, @@ -1761,7 +1775,7 @@ mod vector_grid_tests { let &page_id = pages.get(&page_num).unwrap(); let needed: HashSet = HashSet::from([page_num]); let cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed)); - let ((items, rects, _lines), _has_gid, _rotated, _skipped_invisible) = + let ((items, rects, _lines), _has_gid, _rotated, _skipped_invisible, _) = extract_page_text_items( &doc, page_id, @@ -2499,7 +2513,7 @@ pub fn extract_tables_with_structure_cells_mem( let height = get_page_height(&doc, page_id).unwrap_or(792.0); page_heights.insert(*page_num, height); - let ((mut items, _rects, _lines), _has_gid, coords_rotated, _skipped_invisible) = + let ((mut items, _rects, _lines), _has_gid, coords_rotated, _skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -3302,7 +3316,7 @@ fn detect_tsr_quality_issue( let mut needed: HashSet = HashSet::new(); needed.insert(page_1idx); let font_cmaps = FontCMaps::from_doc_pages_fast(&doc, Some(&needed)); - let ((mut items, _rects, _lines), _has_gid, coords_rotated, _skipped_invisible) = + let ((mut items, _rects, _lines), _has_gid, coords_rotated, _skipped_invisible, _) = extractor::content_stream::extract_page_text_items( &doc, page_id, @@ -4109,7 +4123,7 @@ fn process_document( // (mostly non-alphanumeric), retry with invisible (Tr=3) text included. // This unlocks OCR text layers behind scanned images. if pdf_type == PdfType::Mixed { - if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _)| e) { + if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _, _)| e) { let sample: String = items .iter() .filter(|item| { @@ -4177,8 +4191,26 @@ fn process_document( text_quality_pages, text_quality_reasons_by_page, ) = match extracted { - Some(((items, rects, lines), page_thresholds, gid_encoded_pages)) => { + Some(( + (items, rects, lines), + page_thresholds, + gid_encoded_pages, + operation_limit_pages, + )) => { let mut ocr_reasons_by_page = BTreeMap::new(); + for page in operation_limit_pages { + if options + .page_filter + .as_ref() + .is_none_or(|filter| filter.contains(&page)) + { + add_ocr_reason( + &mut ocr_reasons_by_page, + page, + OCR_REASON_CONTENT_OPERATION_LIMIT, + ); + } + } // For TextBased PDFs with pages flagged for OCR (Identity-H or // Type3 fonts without ToUnicode), check whether the CID-as-Unicode @@ -4414,8 +4446,9 @@ fn process_document( processing_time_ms: start.elapsed_ms(), pages_needing_ocr, ocr_reasons_by_page: { - // Detector reasons (scanned / no_text / vector_text / garbled) merged - // with the markdown-stage garbled detection, deduped per page. + // Detector reasons (scanned / no_text / vector_text / garbled) + // merged with extraction-stage operation-limit and garbled reasons, + // deduped per page. let mut merged = detection_ocr_reasons; merge_ocr_reasons(&mut merged, text_quality_reasons_by_page); page_ocr_reasons_vec(merged) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 1491330b..701d709c 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1968,6 +1968,29 @@ fn test_extract_pages_mem_shifted_cipher_tounicode_needs_ocr() { ); } +#[test] +fn test_extract_pages_mem_reports_content_operation_limit() { + let mut content = Vec::with_capacity(4_200_002); + for _ in 0..2_100_000 { + content.extend_from_slice(b"q\n"); + } + content.extend_from_slice(b"BT /F1 12 Tf 72 720 Td (Hello CAD) Tj ET\n"); + let buf = make_text_pdf( + std::str::from_utf8(&content).expect("test content is ASCII"), + "0 0 612 792", + ); + + let result = extract_pages_markdown_mem(&buf, None).unwrap(); + assert_eq!(result.pages.len(), 1); + assert!(result.pages[0].needs_ocr); + assert!(result.pages[0].markdown.is_empty()); + assert_eq!(result.pages_needing_ocr, vec![1]); + assert_eq!( + result.pages[0].ocr_reason.as_deref(), + Some(pdf_inspector::OCR_REASON_CONTENT_OPERATION_LIMIT) + ); +} + #[test] fn test_extract_regions_mem_multiple_regions_per_page() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap(); From 4229cf6d512d823b645ed6fecaadeb69e9008bd9 Mon Sep 17 00:00:00 2001 From: yzxcj797 <1784931579@qq.com> Date: Tue, 18 Aug 2026 22:38:17 +0800 Subject: [PATCH 2/2] Keep operation-limit diagnostics separate from encoding issues Report budget-skipped pages in pages_needing_ocr without marking the document as encoding-corrupted, and document where the page-level reason is exposed. --- napi/README.md | 2 +- src/lib.rs | 23 +++++++++++++++++++++-- tests/integration_tests.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/napi/README.md b/napi/README.md index 6a8ca91b..9d6f1a00 100644 --- a/napi/README.md +++ b/napi/README.md @@ -94,7 +94,7 @@ console.log(result.confidence) // 0.875 Extract text within bounding-box regions from a PDF. Designed for hybrid OCR pipelines where a layout model detects regions in rendered page images, and this function extracts text from the PDF structure for text-based pages — skipping GPU OCR. -Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues). Known causes are exposed in `ocrReason`; for example, a suspected garbled text layer reports `"suspected_garbled_text"`, and a page skipped by the content-stream safety budget reports `"content_operation_limit"`. +Each region result includes a `needsOcr` flag that signals unreliable extraction (empty text, GID-encoded fonts, garbage text, encoding issues). Known region causes are exposed in `ocrReason`; for example, a suspected garbled text layer reports `"suspected_garbled_text"`. A page skipped by the content-stream safety budget reports `"content_operation_limit"` in the page-level `ocrReasonsByPage` result. ```typescript import { extractTextInRegions } from '@firecrawl/pdf-inspector' diff --git a/src/lib.rs b/src/lib.rs index d10d5084..f53ec3d1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4334,8 +4334,11 @@ fn process_document( )) }; - let enc = !ocr_reasons_by_page.is_empty() - || text_quality.has_encoding_issues + let enc = ocr_reasons_by_page.values().any(|reasons| { + reasons + .iter() + .any(|reason| reason != OCR_REASON_CONTENT_OPERATION_LIMIT) + }) || text_quality.has_encoding_issues || md.as_ref().is_some_and(|m| detect_encoding_issues(m)); ( md, @@ -4407,6 +4410,22 @@ fn process_document( } pages_needing_ocr.sort_unstable(); } + let operation_limit_pages: Vec<_> = text_quality_reasons_by_page + .iter() + .filter(|(_, reasons)| { + reasons + .iter() + .any(|reason| reason == OCR_REASON_CONTENT_OPERATION_LIMIT) + }) + .map(|(page, _)| *page) + .collect(); + for page in operation_limit_pages { + if !pages_needing_ocr.contains(&page) { + pages_needing_ocr.push(page); + } + } + pages_needing_ocr.sort_unstable(); + pages_needing_ocr.dedup(); // Detect sparse extraction: when a TEXT-BASED PDF produces very few // characters per page, the text is likely embedded in images/forms diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 701d709c..8389c5ba 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1991,6 +1991,33 @@ fn test_extract_pages_mem_reports_content_operation_limit() { ); } +#[test] +fn test_process_pdf_mem_reports_operation_limit_without_encoding_issue() { + let mut content = Vec::with_capacity(4_200_002); + for _ in 0..2_100_000 { + content.extend_from_slice(b"q\n"); + } + content.extend_from_slice(b"BT /F1 12 Tf 72 720 Td (Hello CAD) Tj ET\n"); + let buf = make_text_pdf( + std::str::from_utf8(&content).expect("test content is ASCII"), + "0 0 612 792", + ); + + let result = process_pdf_mem(&buf).unwrap(); + + assert_eq!(result.page_count, 1); + assert_eq!(result.pages_needing_ocr, vec![1]); + assert!(!result.has_encoding_issues); + assert_eq!( + result + .ocr_reasons_by_page + .iter() + .flat_map(|reason| reason.reasons.iter().cloned()) + .collect::>(), + vec![pdf_inspector::OCR_REASON_CONTENT_OPERATION_LIMIT.to_string()] + ); +} + #[test] fn test_extract_regions_mem_multiple_regions_per_page() { let buf = std::fs::read("tests/fixtures/nexo-price-en.pdf").unwrap();