diff --git a/.github/workflows/ci-wasm.yml b/.github/workflows/ci-wasm.yml index 8d80c649..c2767730 100644 --- a/.github/workflows/ci-wasm.yml +++ b/.github/workflows/ci-wasm.yml @@ -113,7 +113,13 @@ jobs: path: packages/wasm/pkg/ - name: Install Miniflare - run: npm install miniflare@latest + # Pinned, NOT @latest: miniflare@latest currently resolves to a 5.x + # alpha whose `new Miniflare()` options schema replaced the 4.x + # single-worker form (`modules: [...]`) used by + # scripts/edge-compat/wasm-test.mjs with a nested `workers: [...]` + # shape. There is no root package.json/lockfile in this repo, so the + # version has to live here. Bump deliberately, re-running the test. + run: npm install --no-save miniflare@4.20260730.0 - name: Run edge runtime parse test run: node scripts/edge-compat/wasm-test.mjs diff --git a/.gitignore b/.gitignore index a620ebcc..aee1078f 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ temp/ !integration_tests_data/receipt.png !integration_tests_data/sample3.docx.doc !integration_tests_data/sample.pdf +!integration_tests_data/filled_acroform.pdf deno.json # Python files diff --git a/Cargo.lock b/Cargo.lock index e067d842..01654139 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2016,7 +2016,7 @@ checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "liteparse" -version = "2.10.1" +version = "2.12.0" dependencies = [ "blake3", "clap", @@ -2027,6 +2027,7 @@ dependencies = [ "liteparse-pdfium", "liteparse-pdfium-sys", "lopdf", + "memchr", "oar-ocr", "ordered-float", "regex", @@ -2047,7 +2048,7 @@ dependencies = [ [[package]] name = "liteparse-napi" -version = "2.10.1" +version = "2.12.0" dependencies = [ "image", "liteparse", @@ -2063,7 +2064,7 @@ dependencies = [ [[package]] name = "liteparse-pdfium" -version = "1.4.0" +version = "1.7.0" dependencies = [ "blake3", "liteparse-pdfium-sys", @@ -2071,7 +2072,7 @@ dependencies = [ [[package]] name = "liteparse-pdfium-sys" -version = "1.4.0" +version = "1.7.0" dependencies = [ "bindgen", "flate2", @@ -2082,7 +2083,7 @@ dependencies = [ [[package]] name = "liteparse-python" -version = "2.10.1" +version = "2.12.0" dependencies = [ "anyhow", "clap", @@ -2098,7 +2099,7 @@ dependencies = [ [[package]] name = "liteparse-wasm" -version = "2.10.1" +version = "2.12.0" dependencies = [ "console_error_panic_hook", "js-sys", diff --git a/README.md b/README.md index ff0a8a07..4f53e235 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,8 @@ npx skills add run-llama/llamaparse-agent-skills --skill liteparse Or copy-pasting the [`SKILL.md`](https://github.com/run-llama/llamaparse-agent-skills/blob/main/skills/liteparse/SKILL.md) file to your own skills setup. +See the [Agent Skill guide](https://developers.llamaindex.ai/liteparse/guides/agent-skill/?utm_source=github&utm_medium=liteparse) for requirements and usage patterns. + ## CLI Usage The CLI is the same across all installations (`npm`, `pip`, `cargo install`). @@ -257,7 +259,18 @@ link annotations. The field is absent by default; enabled untagged pages contain Parse results (Rust/Node/Python APIs) carry the document's `/Info` `creator` and `producer` entries when present; these are API-only and never appear in -CLI JSON output. Enable `--extract-content-bounds` (Rust/Python +CLI JSON output. Enable `extract_document_metadata` (JavaScript/WASM +`extractDocumentMetadata`) to add `doc_meta`/`docMeta`, a provenance object +with the `/Info` creation/modification dates, PDF version and encryption +permissions, signature state, incremental-save markers, trailer ID comparison, +the document catalog's XMP packet (capped at 64 KiB, with `xmp_truncated` +when it was cut), and source file size. It is off by default because it +streams the whole source file once; it is absent for inputs converted from a +non-PDF format, where the facts would describe the intermediate PDF rather +than your file. `xmp` needs a structural parse of the document, so it is +skipped (left absent) for sources over 16 MiB and in WASM builds — the other +fields are unaffected. +Enable `--extract-content-bounds` (Rust/Python `extract_content_bounds`, JavaScript/WASM `extractContentBounds`) to add a per-page `content_bounds`: the union bbox of the page's top-level content objects in viewport coords (absent for empty pages). Enable diff --git a/crates/liteparse-napi/Cargo.toml b/crates/liteparse-napi/Cargo.toml index 7d6c53e8..10fc35b0 100644 --- a/crates/liteparse-napi/Cargo.toml +++ b/crates/liteparse-napi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "liteparse-napi" -version = "2.10.1" +version = "2.12.0" edition.workspace = true license.workspace = true repository.workspace = true @@ -14,11 +14,11 @@ default = ["tesseract"] tesseract = ["liteparse/tesseract"] [dependencies] -liteparse = { package = "liteparse", version = "2.10.1", path = "../liteparse", default-features = false } -pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.4.0", path = "../pdfium-sys" } +liteparse = { package = "liteparse", version = "2.12.0", path = "../liteparse", default-features = false } +pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.5.0", path = "../pdfium-sys" } napi = { version = "2", features = ["async", "serde-json", "napi9"] } napi-derive = "2" -pdfium = { package = "liteparse-pdfium", version = "1.4.0", path = "../pdfium" } +pdfium = { package = "liteparse-pdfium", version = "1.5.0", path = "../pdfium" } image = { version = "0.25", default-features = false, features = ["png"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/liteparse-napi/src/lib.rs b/crates/liteparse-napi/src/lib.rs index 27a725a5..c7e7ba31 100644 --- a/crates/liteparse-napi/src/lib.rs +++ b/crates/liteparse-napi/src/lib.rs @@ -4,8 +4,8 @@ use napi_derive::napi; mod types; use types::{ - JsLiteParseConfig, JsPageComplexityStats, JsPageInput, JsParseResult, JsScreenshotResult, - JsTextItem, + JsLiteParseConfig, JsPageComplexityStats, JsPageInput, JsParseBatch, JsParseResult, + JsScreenshotResult, JsTextItem, }; /// Main LiteParse parser class. @@ -48,6 +48,43 @@ impl LiteParse { Ok(JsParseResult::from_rust(&result, &self.config)) } + /// Open a document for bounded-memory batch parsing. Internal plumbing + /// for the JS wrapper's `parseBatches()` — prefer that; it also closes + /// the session for you. + /// + /// Converts a non-PDF source once, then yields `batchSize` pages at a time + /// via `nextBatch()` (default 25). Cross-page passes (repeated + /// header/footer removal, image deduplication) see only the pages in their + /// own batch, so output can differ from a whole-document `parse()`. + #[napi] + pub async fn open_batch_session( + &self, + input: Either, + batch_size: Option, + ) -> Result { + use liteparse::types::PdfInput; + + let pdf_input = match input { + Either::A(path) => PdfInput::Path(path), + Either::B(buf) => PdfInput::Bytes(buf.to_vec()), + }; + let batch_size = batch_size + .map(|v| v as usize) + .unwrap_or(liteparse::DEFAULT_PAGE_BATCH_SIZE); + + let session = self + .inner + .open_batch_session(pdf_input, batch_size) + .await + .map_err(|e| Error::from_reason(e.to_string()))?; + + Ok(ParseSession { + total_pages: session.total_pages(), + inner: std::sync::Arc::new(tokio::sync::Mutex::new(Some(session))), + config: self.config.clone(), + }) + } + /// Parse from pre-extracted pages, skipping PDFium text extraction. /// /// The caller supplies pages already populated with text items in viewport @@ -134,6 +171,67 @@ impl LiteParse { } } +/// A document opened once and parsed in bounded page batches. Internal +/// plumbing for the JS wrapper's `parseBatches()` — prefer that. +/// +/// Created by `LiteParse.openBatchSession()`. The converted-PDF temporary +/// file for a non-PDF source lives as long as the session, so conversion is +/// paid once no matter how many batches are consumed. Call `close()` when +/// abandoning the session early — otherwise that temp file waits for GC. +#[napi] +pub struct ParseSession { + /// The core session is `&mut` per batch, but napi hands out `&self`, so + /// the mutation is serialized here. Concurrent `nextBatch()` calls queue + /// rather than interleave, which also keeps batch order well-defined. + /// `None` after `close()`. + inner: std::sync::Arc>>, + config: liteparse::config::LiteParseConfig, + total_pages: u32, +} + +#[napi] +impl ParseSession { + /// Total pages in the source document, before `maxPages` or batching. + #[napi(getter)] + pub fn total_pages(&self) -> u32 { + self.total_pages + } + + /// Parse and return the next batch, or `null` once every page within + /// `maxPages` has been yielded. Rejects if the session is closed. + #[napi] + pub async fn next_batch(&self) -> Result> { + let inner = self.inner.clone(); + let mut session = inner.lock().await; + let batch = session + .as_mut() + .ok_or_else(|| Error::from_reason("session is closed"))? + .next_batch() + .await + .map_err(|e| Error::from_reason(e.to_string()))?; + + Ok(batch.map(|batch| JsParseBatch { + start_page: batch.start_page, + end_page: batch.end_page, + result: JsParseResult::from_rust(&batch.result, &self.config), + })) + } + + /// Release the session's resources now — most importantly the converted + /// temporary PDF for a non-PDF source, which otherwise lives until the + /// JS object is garbage collected. Idempotent; `nextBatch()` rejects + /// afterwards. + #[napi] + pub async fn close(&self) -> Result<()> { + let inner = self.inner.clone(); + let mut session = inner.lock().await; + // Dropping the core session drops the conversion guard, which + // removes the temp file. + session.take(); + Ok(()) + } +} + /// Search text items for phrase matches, returning merged items with combined bounding boxes. #[napi] pub fn search_items( diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs index d8a9a2fa..88e2bb7c 100644 --- a/crates/liteparse-napi/src/types.rs +++ b/crates/liteparse-napi/src/types.rs @@ -32,6 +32,12 @@ pub struct JsLiteParseConfig { pub max_pages: Option, /// Specific pages to parse (e.g., "1-5,10,15-20"). pub target_pages: Option, + /// Render parsed pages to PNG and return them in `ParseResult.screenshots`. + /// Default false; PNG payloads can be large. + pub extract_screenshots: Option, + /// Continue after page-level extraction failures and return them in + /// `ParseResult.pageErrors`. Default false. + pub continue_on_page_error: Option, /// DPI for rendering pages (used for OCR and screenshots). pub dpi: Option, /// Output format: "json", "text", or "markdown". @@ -68,6 +74,10 @@ pub struct JsLiteParseConfig { /// Extract raw XFA packets (name + XML content) into /// `ParseResult.xfaPackets`. Default false. pub extract_xfa_packets: Option, + /// Collect document provenance metadata into `ParseResult.docMeta`. + /// Default false: it streams the whole source file once. Absent for + /// inputs converted from a non-PDF format. + pub extract_document_metadata: Option, /// Emit each page's `contentBounds` (union bbox of top-level content /// objects, viewport coords). Default false. pub extract_content_bounds: Option, @@ -141,6 +151,12 @@ impl JsLiteParseConfig { if let Some(v) = self.target_pages { cfg.target_pages = Some(v); } + if let Some(v) = self.extract_screenshots { + cfg.extract_screenshots = v; + } + if let Some(v) = self.continue_on_page_error { + cfg.continue_on_page_error = v; + } if let Some(v) = self.dpi { cfg.dpi = v as f32; } @@ -194,6 +210,9 @@ impl JsLiteParseConfig { if let Some(v) = self.extract_xfa_packets { cfg.extract_xfa_packets = v; } + if let Some(v) = self.extract_document_metadata { + cfg.extract_document_metadata = v; + } if let Some(v) = self.extract_content_bounds { cfg.extract_content_bounds = v; } @@ -248,6 +267,8 @@ impl JsLiteParseConfig { tessdata_path: cfg.tessdata_path.clone(), max_pages: Some(cfg.max_pages as u32), target_pages: cfg.target_pages.clone(), + extract_screenshots: Some(cfg.extract_screenshots), + continue_on_page_error: Some(cfg.continue_on_page_error), dpi: Some(cfg.dpi as f64), output_format: Some(match cfg.output_format { OutputFormat::Json => "json".to_string(), @@ -271,6 +292,7 @@ impl JsLiteParseConfig { extract_form_fields: Some(cfg.extract_form_fields), extract_structure_tree: Some(cfg.extract_structure_tree), extract_xfa_packets: Some(cfg.extract_xfa_packets), + extract_document_metadata: Some(cfg.extract_document_metadata), extract_content_bounds: Some(cfg.extract_content_bounds), detect_screenshot_rects: Some(cfg.detect_screenshot_rects), render_form_fields: Some(cfg.render_form_fields), @@ -873,19 +895,85 @@ impl JsParsedPage { #[napi(object)] #[derive(Clone)] pub struct JsParseResult { + /// Total source-document pages before target/max-page filtering. + pub total_pages: u32, pub pages: Vec, + pub page_errors: Vec, pub text: String, pub images: Vec, + pub screenshots: Vec, pub image_error_count: u32, pub form_type: Option, /// The document's `/Info` `Creator` entry, when present. pub creator: Option, /// The document's `/Info` `Producer` entry, when present. pub producer: Option, + /// Document-level provenance metadata; present only when + /// `extractDocumentMetadata` is enabled and the input was a real PDF. + pub doc_meta: Option, /// Raw XFA packets; present only when `extractXfaPackets` is enabled. pub xfa_packets: Option>, } +/// One batch of pages from a `ParseSession`. +#[napi(object)] +pub struct JsParseBatch { + /// First source page in this batch, 1-indexed. + pub start_page: u32, + /// Last source page in this batch, 1-indexed and inclusive. + pub end_page: u32, + /// The pages in `startPage..=endPage`, as an ordinary parse result. + pub result: JsParseResult, +} + +#[napi(object)] +#[derive(Clone)] +pub struct JsPageError { + pub page_num: u32, + pub message: String, +} + +#[napi(object)] +#[derive(Clone)] +pub struct JsDocumentMetadata { + pub creation_date: Option, + pub mod_date: Option, + pub file_version: Option, + pub is_encrypted: Option, + pub security_handler_revision: Option, + pub permissions: Option, + pub eof_section_count: Option, + pub startxref_count: Option, + pub trailer_id_pair_differs: Option, + pub raw_file_size: Option, + pub xmp: Option, + /// True when the catalog's XMP stream exceeded the 64 KiB cap. + pub xmp_truncated: Option, + pub signature_count: Option, + pub signature_byte_range_reaches_eof: Option, +} + +impl JsDocumentMetadata { + fn from_rust(metadata: &liteparse::types::DocumentMetadata) -> Self { + Self { + creation_date: metadata.creation_date.clone(), + mod_date: metadata.mod_date.clone(), + file_version: metadata.file_version, + is_encrypted: metadata.is_encrypted, + security_handler_revision: metadata.security_handler_revision, + permissions: metadata.permissions.map(|value| value as f64), + eof_section_count: metadata.eof_section_count, + startxref_count: metadata.startxref_count, + trailer_id_pair_differs: metadata.trailer_id_pair_differs, + raw_file_size: metadata.raw_file_size.map(|value| value as f64), + xmp: metadata.xmp.clone(), + xmp_truncated: metadata.xmp_truncated, + signature_count: metadata.signature_count, + signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, + } + } +} + /// One raw packet from an XFA form document's `/XFA` array. #[napi(object)] #[derive(Clone)] @@ -979,6 +1067,23 @@ impl JsScreenshotRect { } } +impl JsScreenshotResult { + pub fn from_rust(result: &liteparse::parser::ScreenshotResult) -> Self { + Self { + page_num: result.page_num, + width: result.width, + height: result.height, + image_buffer: result.image_bytes.clone().into(), + is_solid_fill: result.is_solid_fill, + rects: result + .rects + .iter() + .map(JsScreenshotRect::from_rust) + .collect(), + } + } +} + #[napi(object)] #[derive(Clone)] pub struct JsLayoutComplexityStats { @@ -1073,16 +1178,26 @@ impl JsPageComplexityStats { impl JsParseResult { pub fn from_rust(result: &ParseResult, config: &LiteParseConfig) -> Self { Self { + total_pages: result.total_pages, pages: result .pages .iter() .map(|page| JsParsedPage::from_rust(page, config.extract_text_metadata)) .collect(), + page_errors: result + .page_errors + .iter() + .map(|error| JsPageError { + page_num: error.page_number, + message: error.message.clone(), + }) + .collect(), text: result.text.clone(), image_error_count: result.image_error_count, form_type: result.form_type, creator: result.creator.clone(), producer: result.producer.clone(), + doc_meta: result.doc_meta.as_ref().map(JsDocumentMetadata::from_rust), xfa_packets: result .xfa_packets .as_ref() @@ -1109,6 +1224,11 @@ impl JsParseResult { bytes: img.bytes.as_slice().to_vec().into(), }) .collect(), + screenshots: result + .screenshots + .iter() + .map(JsScreenshotResult::from_rust) + .collect(), } } } diff --git a/crates/liteparse-python/Cargo.toml b/crates/liteparse-python/Cargo.toml index 722c8559..599d412e 100644 --- a/crates/liteparse-python/Cargo.toml +++ b/crates/liteparse-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "liteparse-python" -version = "2.10.1" +version = "2.12.0" edition.workspace = true license.workspace = true repository.workspace = true @@ -15,9 +15,9 @@ default = ["tesseract"] tesseract = ["liteparse/tesseract"] [dependencies] -liteparse = { package = "liteparse", version = "2.10.1", path = "../liteparse", default-features = false } -pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.4.0", path = "../pdfium-sys" } -pdfium = { package = "liteparse-pdfium", version = "1.4.0", path = "../pdfium" } +liteparse = { package = "liteparse", version = "2.12.0", path = "../liteparse", default-features = false } +pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.5.0", path = "../pdfium-sys" } +pdfium = { package = "liteparse-pdfium", version = "1.5.0", path = "../pdfium" } clap = { version = "4.5.55", features = ["derive"] } pyo3 = { version = "0.29", features = ["extension-module"] } anyhow = "1.0.102" diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rs index 2a7c1284..69f1cc8d 100644 --- a/crates/liteparse-python/src/cli.rs +++ b/crates/liteparse-python/src/cli.rs @@ -51,6 +51,9 @@ struct ParseCommand { max_pages: usize, #[arg(long)] target_pages: Option, + /// Continue after page-level extraction errors and report them in JSON. + #[arg(long)] + continue_on_page_error: bool, #[arg(long, default_value = "150")] dpi: f32, #[arg(long)] @@ -269,6 +272,7 @@ pub fn run_cli(args: Vec) -> Result<(), Box> { tessdata_path: cmd.tessdata_path, max_pages: cmd.max_pages, target_pages: cmd.target_pages, + continue_on_page_error: cmd.continue_on_page_error, dpi: cmd.dpi, output_format: format, preserve_very_small_text: cmd.preserve_small_text, @@ -300,6 +304,14 @@ pub fn run_cli(args: Vec) -> Result<(), Box> { } else { rt.block_on(lp.parse(&cmd.file))? }; + // JSON output carries `page_errors` itself; text/markdown would + // silently omit the failed pages, so always surface them on stderr. + for error in &result.page_errors { + eprintln!( + "[liteparse] page {} failed to extract and was skipped: {}", + error.page_number, error.message + ); + } let formatted = match lp.config().output_format { OutputFormat::Json => { json::format_json_result(&result, lp.config().extract_text_metadata)? diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs index e15a0c14..c465df78 100644 --- a/crates/liteparse-python/src/lib.rs +++ b/crates/liteparse-python/src/lib.rs @@ -596,6 +596,8 @@ impl PyParsedPage { #[pyclass(frozen, from_py_object)] #[derive(Clone)] struct PyParseResult { + #[pyo3(get)] + total_pages: u32, #[pyo3(get)] pages: Vec, #[pyo3(get)] @@ -603,17 +605,86 @@ struct PyParseResult { #[pyo3(get)] images: Vec, #[pyo3(get)] + screenshots: Vec, + #[pyo3(get)] image_error_count: u32, #[pyo3(get)] + page_errors: Vec, + #[pyo3(get)] form_type: Option, #[pyo3(get)] creator: Option, #[pyo3(get)] producer: Option, #[pyo3(get)] + doc_meta: Option, + #[pyo3(get)] xfa_packets: Option>, } +#[pyclass(frozen, from_py_object)] +#[derive(Clone)] +struct PyPageError { + #[pyo3(get)] + page_num: u32, + #[pyo3(get)] + message: String, +} + +#[pyclass(frozen, from_py_object)] +#[derive(Clone)] +struct PyDocumentMetadata { + #[pyo3(get)] + creation_date: Option, + #[pyo3(get)] + mod_date: Option, + #[pyo3(get)] + file_version: Option, + #[pyo3(get)] + is_encrypted: Option, + #[pyo3(get)] + security_handler_revision: Option, + #[pyo3(get)] + permissions: Option, + #[pyo3(get)] + eof_section_count: Option, + #[pyo3(get)] + startxref_count: Option, + #[pyo3(get)] + trailer_id_pair_differs: Option, + #[pyo3(get)] + raw_file_size: Option, + #[pyo3(get)] + xmp: Option, + #[pyo3(get)] + xmp_truncated: Option, + #[pyo3(get)] + signature_count: Option, + #[pyo3(get)] + signature_byte_range_reaches_eof: Option, +} + +impl From for PyDocumentMetadata { + fn from(metadata: liteparse::types::DocumentMetadata) -> Self { + Self { + creation_date: metadata.creation_date, + mod_date: metadata.mod_date, + file_version: metadata.file_version, + is_encrypted: metadata.is_encrypted, + security_handler_revision: metadata.security_handler_revision, + permissions: metadata.permissions, + eof_section_count: metadata.eof_section_count, + startxref_count: metadata.startxref_count, + trailer_id_pair_differs: metadata.trailer_id_pair_differs, + raw_file_size: metadata.raw_file_size, + xmp: metadata.xmp, + xmp_truncated: metadata.xmp_truncated, + signature_count: metadata.signature_count, + signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, + } + } +} + /// One raw packet from an XFA form document's `/XFA` array. #[pyclass(frozen, from_py_object)] #[derive(Clone)] @@ -662,6 +733,7 @@ impl PyParseResult { impl PyParseResult { fn from_rust(result: liteparse::parser::ParseResult, extract_text_metadata: bool) -> Self { Self { + total_pages: result.total_pages, pages: result .pages .into_iter() @@ -673,10 +745,24 @@ impl PyParseResult { .into_iter() .map(PyExtractedImage::from_rust) .collect(), + screenshots: result + .screenshots + .into_iter() + .map(PyScreenshotResult::from_rust) + .collect(), image_error_count: result.image_error_count, + page_errors: result + .page_errors + .into_iter() + .map(|error| PyPageError { + page_num: error.page_number, + message: error.message, + }) + .collect(), form_type: result.form_type, creator: result.creator, producer: result.producer, + doc_meta: result.doc_meta.map(Into::into), xfa_packets: result.xfa_packets.map(|packets| { packets .into_iter() @@ -817,6 +903,30 @@ impl PyScreenshotRect { } } +impl PyScreenshotResult { + fn from_rust(result: liteparse::parser::ScreenshotResult) -> Self { + Self { + page_num: result.page_num, + width: result.width, + height: result.height, + image_buffer: result.image_bytes, + is_solid_fill: result.is_solid_fill, + rects: result + .rects + .into_iter() + .map(|rect| PyScreenshotRect { + x: rect.x as f64, + y: rect.y as f64, + width: rect.width as f64, + height: rect.height as f64, + color: rect.color, + is_line: rect.is_line, + }) + .collect(), + } + } +} + #[pymethods] impl PyScreenshotResult { #[getter] @@ -987,6 +1097,10 @@ struct PyLiteParseConfig { #[pyo3(get)] target_pages: Option, #[pyo3(get)] + extract_screenshots: bool, + #[pyo3(get)] + continue_on_page_error: bool, + #[pyo3(get)] dpi: f32, #[pyo3(get)] output_format: String, @@ -1013,6 +1127,8 @@ struct PyLiteParseConfig { #[pyo3(get)] extract_xfa_packets: bool, #[pyo3(get)] + extract_document_metadata: bool, + #[pyo3(get)] extract_content_bounds: bool, #[pyo3(get)] detect_screenshot_rects: bool, @@ -1064,6 +1180,8 @@ impl PyLiteParseConfig { tessdata_path: cfg.tessdata_path.clone(), max_pages: cfg.max_pages, target_pages: cfg.target_pages.clone(), + extract_screenshots: cfg.extract_screenshots, + continue_on_page_error: cfg.continue_on_page_error, dpi: cfg.dpi, output_format: match cfg.output_format { OutputFormat::Json => "json".to_string(), @@ -1085,6 +1203,7 @@ impl PyLiteParseConfig { extract_form_fields: cfg.extract_form_fields, extract_structure_tree: cfg.extract_structure_tree, extract_xfa_packets: cfg.extract_xfa_packets, + extract_document_metadata: cfg.extract_document_metadata, extract_content_bounds: cfg.extract_content_bounds, detect_screenshot_rects: cfg.detect_screenshot_rects, render_form_fields: cfg.render_form_fields, @@ -1105,6 +1224,76 @@ impl PyLiteParseConfig { } } +// --------------------------------------------------------------------------- +// Batch parsing +// --------------------------------------------------------------------------- + +/// One batch of pages from a `_ParseSession`. Internal plumbing for the +/// wrapper's `parse_batches()`, which converts it into the public +/// `liteparse.types.ParseBatch` dataclass — the underscore name keeps the +/// two from colliding. +#[pyclass(frozen, name = "_ParseBatch", skip_from_py_object)] +#[derive(Clone)] +struct PyParseBatch { + /// First source page in this batch (1-indexed). + #[pyo3(get)] + start_page: u32, + /// Last source page in this batch (1-indexed, inclusive). + #[pyo3(get)] + end_page: u32, + /// The pages in `start_page..=end_page`, as an ordinary parse result. + #[pyo3(get)] + result: PyParseResult, +} + +/// A document opened once and parsed in bounded page batches. Internal +/// plumbing for the wrapper's `parse_batches()` — prefer that. +/// +/// Iterate it directly to consume every batch: +/// +/// for batch in parser.open_batch_session("large.pdf", batch_size=20): +/// handle(batch.result.pages) +#[pyclass(name = "_ParseSession", unsendable)] +struct PyParseSession { + inner: liteparse::ParseSession, + runtime: std::sync::Arc, + extract_text_metadata: bool, +} + +#[pymethods] +impl PyParseSession { + /// Total pages in the source document, before `max_pages` or batching. + #[getter] + fn total_pages(&self) -> u32 { + self.inner.total_pages() + } + + /// Parse and return the next batch, or `None` once every page within + /// `max_pages` has been yielded. + fn next_batch(&mut self, py: Python<'_>) -> PyResult> { + // Releasing the GIL keeps other Python threads running while PDFium + // extraction and grid projection execute, matching `parse()`. + let batch = py + .detach(|| self.runtime.block_on(self.inner.next_batch())) + .map_err(|e| PyErr::new::(e.to_string()))?; + + Ok(batch.map(|batch| PyParseBatch { + start_page: batch.start_page, + end_page: batch.end_page, + result: PyParseResult::from_rust(batch.result, self.extract_text_metadata), + })) + } + + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + /// Returning `None` raises `StopIteration`, ending the loop. + fn __next__(&mut self, py: Python<'_>) -> PyResult> { + self.next_batch(py) + } +} + // --------------------------------------------------------------------------- // Main LiteParse class // --------------------------------------------------------------------------- @@ -1113,7 +1302,7 @@ impl PyLiteParseConfig { struct LiteParse { inner: liteparse::parser::LiteParse, config: LiteParseConfig, - runtime: tokio::runtime::Runtime, + runtime: std::sync::Arc, } #[pymethods] @@ -1128,6 +1317,8 @@ impl LiteParse { tessdata_path = None, max_pages = None, target_pages = None, + extract_screenshots = None, + continue_on_page_error = None, dpi = None, output_format = None, preserve_very_small_text = None, @@ -1143,6 +1334,7 @@ impl LiteParse { extract_form_fields = None, extract_structure_tree = None, extract_xfa_packets = None, + extract_document_metadata = None, extract_content_bounds = None, detect_screenshot_rects = None, render_form_fields = None, @@ -1163,6 +1355,8 @@ impl LiteParse { tessdata_path: Option, max_pages: Option, target_pages: Option, + extract_screenshots: Option, + continue_on_page_error: Option, dpi: Option, output_format: Option, preserve_very_small_text: Option, @@ -1178,6 +1372,7 @@ impl LiteParse { extract_form_fields: Option, extract_structure_tree: Option, extract_xfa_packets: Option, + extract_document_metadata: Option, extract_content_bounds: Option, detect_screenshot_rects: Option, render_form_fields: Option, @@ -1212,6 +1407,12 @@ impl LiteParse { if let Some(v) = target_pages { cfg.target_pages = Some(v); } + if let Some(v) = extract_screenshots { + cfg.extract_screenshots = v; + } + if let Some(v) = continue_on_page_error { + cfg.continue_on_page_error = v; + } if let Some(v) = dpi { cfg.dpi = v; } @@ -1265,6 +1466,9 @@ impl LiteParse { if let Some(v) = extract_xfa_packets { cfg.extract_xfa_packets = v; } + if let Some(v) = extract_document_metadata { + cfg.extract_document_metadata = v; + } if let Some(v) = extract_content_bounds { cfg.extract_content_bounds = v; } @@ -1305,8 +1509,10 @@ impl LiteParse { } let inner = liteparse::parser::LiteParse::new(cfg.clone()); - let runtime = tokio::runtime::Runtime::new() - .map_err(|e| PyErr::new::(e.to_string()))?; + let runtime = std::sync::Arc::new( + tokio::runtime::Runtime::new() + .map_err(|e| PyErr::new::(e.to_string()))?, + ); Ok(Self { inner, @@ -1327,6 +1533,35 @@ impl LiteParse { )) } + /// Open a document from a file path for bounded-memory batch parsing. + /// Internal plumbing for the wrapper's `parse_batches()` — prefer that. + /// + /// Converts a non-PDF source once and returns a `_ParseSession` yielding + /// `batch_size` pages at a time. Cross-page passes (repeated header/footer + /// removal, image deduplication) see only the pages in their own batch, so + /// output can differ from a whole-document `parse()`. + #[pyo3(signature = (input, batch_size = None))] + fn open_batch_session( + &self, + py: Python<'_>, + input: String, + batch_size: Option, + ) -> PyResult { + self.open_session(py, PdfInput::Path(input), batch_size) + } + + /// Open a document from raw bytes for bounded-memory batch parsing. + /// Internal plumbing for the wrapper's `parse_batches()` — prefer that. + #[pyo3(signature = (data, batch_size = None))] + fn open_batch_session_bytes( + &self, + py: Python<'_>, + data: Vec, + batch_size: Option, + ) -> PyResult { + self.open_session(py, PdfInput::Bytes(data), batch_size) + } + /// Parse a document from raw bytes. fn parse_bytes(&self, py: Python<'_>, data: Vec) -> PyResult { let pdf_input = PdfInput::Bytes(data); @@ -1419,6 +1654,31 @@ impl LiteParse { } } +impl LiteParse { + /// Shared body of `open_batch_session` / `open_batch_session_bytes`. Not + /// a `#[pymethods]` entry, so it stays off the Python surface. + fn open_session( + &self, + py: Python<'_>, + input: PdfInput, + batch_size: Option, + ) -> PyResult { + let batch_size = batch_size.unwrap_or(liteparse::DEFAULT_PAGE_BATCH_SIZE); + let session = py + .detach(|| { + self.runtime + .block_on(self.inner.open_batch_session(input, batch_size)) + }) + .map_err(|e| PyErr::new::(e.to_string()))?; + + Ok(PyParseSession { + inner: session, + runtime: self.runtime.clone(), + extract_text_metadata: self.config.extract_text_metadata, + }) + } +} + // --------------------------------------------------------------------------- // Module // --------------------------------------------------------------------------- @@ -1491,6 +1751,10 @@ fn _liteparse(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/crates/liteparse-wasm/Cargo.toml b/crates/liteparse-wasm/Cargo.toml index cb4f8155..2e07417d 100644 --- a/crates/liteparse-wasm/Cargo.toml +++ b/crates/liteparse-wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "liteparse-wasm" -version = "2.10.1" +version = "2.12.0" edition.workspace = true license.workspace = true repository.workspace = true @@ -26,7 +26,7 @@ console_error_panic_hook = { version = "0.1", optional = true } # fallback setjmp/longjmp stubs in wasi_stubs.rs. Only the wasm target links # pdfium statically, so scope it there. [target.'cfg(target_arch = "wasm32")'.dependencies] -pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.4.0", path = "../pdfium-sys" } +pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.5.0", path = "../pdfium-sys" } [features] default = ["panic_hook"] diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs index 9532a9fd..37f2cb07 100644 --- a/crates/liteparse-wasm/src/lib.rs +++ b/crates/liteparse-wasm/src/lib.rs @@ -49,6 +49,15 @@ pub struct LiteParseConfig { tessdata_path: Option, max_pages: Option, target_pages: Option, + /// Skip page-level PDF extraction failures and report them in + /// `ParseResult.pageErrors`. Document-level failures remain fatal. + continue_on_page_error: Option, + /// Render parsed pages to PNG and return them in + /// `ParseResult.screenshots`. Default false; PNG payloads can be large. + extract_screenshots: Option, + /// Scan rendered screenshots for solid rectangles/lines and attach + /// them to each screenshot result. Default false. + detect_screenshot_rects: Option, dpi: Option, #[tsify(type = "\"json\" | \"text\" | \"markdown\" | \"md\"")] output_format: Option, @@ -65,6 +74,9 @@ pub struct LiteParseConfig { /// Extract raw XFA packets (name + XML content) into /// `ParseResult.xfaPackets`. Default false. extract_xfa_packets: Option, + /// Collect document provenance metadata into `ParseResult.docMeta`. + /// Default false: it streams the whole source file once. + extract_document_metadata: Option, /// Emit each page's `contentBounds` (union bbox of top-level content /// objects, viewport coords). Default false. extract_content_bounds: Option, @@ -132,6 +144,15 @@ impl LiteParseConfig { if self.target_pages.is_some() { cfg.target_pages = self.target_pages; } + if let Some(v) = self.continue_on_page_error { + cfg.continue_on_page_error = v; + } + if let Some(v) = self.extract_screenshots { + cfg.extract_screenshots = v; + } + if let Some(v) = self.detect_screenshot_rects { + cfg.detect_screenshot_rects = v; + } if let Some(v) = self.dpi { cfg.dpi = v; } @@ -176,6 +197,9 @@ impl LiteParseConfig { if let Some(v) = self.extract_xfa_packets { cfg.extract_xfa_packets = v; } + if let Some(v) = self.extract_document_metadata { + cfg.extract_document_metadata = v; + } if let Some(v) = self.extract_content_bounds { cfg.extract_content_bounds = v; } @@ -237,6 +261,9 @@ impl LiteParseConfig { tessdata_path: cfg.tessdata_path.clone(), max_pages: Some(cfg.max_pages), target_pages: cfg.target_pages.clone(), + continue_on_page_error: Some(cfg.continue_on_page_error), + extract_screenshots: Some(cfg.extract_screenshots), + detect_screenshot_rects: Some(cfg.detect_screenshot_rects), dpi: Some(cfg.dpi), output_format: Some(match cfg.output_format { OutputFormat::Json => "json".into(), @@ -255,6 +282,7 @@ impl LiteParseConfig { extract_form_fields: Some(cfg.extract_form_fields), extract_structure_tree: Some(cfg.extract_structure_tree), extract_xfa_packets: Some(cfg.extract_xfa_packets), + extract_document_metadata: Some(cfg.extract_document_metadata), extract_content_bounds: Some(cfg.extract_content_bounds), ocr_failure_fatal: Some(cfg.ocr_failure_fatal), ocr_hedge_delays_ms: Some(cfg.ocr_hedge_delays_ms.clone()), @@ -607,10 +635,16 @@ impl StructureTreeElement { #[tsify(into_wasm_abi)] #[serde(rename_all = "camelCase")] pub struct ParseResult { + /// Total source-document pages before target/max-page filtering. + pub total_pages: u32, pub pages: Vec, pub text: String, pub images: Vec, + /// Page screenshots encoded as PNG. Empty unless `extractScreenshots` + /// is enabled. + pub screenshots: Vec, pub image_error_count: u32, + pub page_errors: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub form_type: Option, /// The document's `/Info` `Creator` entry, when present. @@ -619,11 +653,112 @@ pub struct ParseResult { /// The document's `/Info` `Producer` entry, when present. #[serde(skip_serializing_if = "Option::is_none")] pub producer: Option, + /// Document-level provenance metadata; present only when + /// `extractDocumentMetadata` is enabled and the input was a real PDF. + #[serde(skip_serializing_if = "Option::is_none")] + pub doc_meta: Option, /// Raw XFA packets; present only when `extractXfaPackets` is enabled. #[serde(skip_serializing_if = "Option::is_none")] pub xfa_packets: Option>, } +#[derive(Serialize, Tsify)] +#[tsify(into_wasm_abi)] +#[serde(rename_all = "camelCase")] +pub struct PageError { + pub page_num: u32, + pub message: String, +} + +/// One page rendered to PNG, plus raster-derived signals. +#[derive(Serialize, Tsify)] +#[tsify(into_wasm_abi)] +#[serde(rename_all = "camelCase")] +pub struct ScreenshotResult { + pub page_num: u32, + pub width: u32, + pub height: u32, + /// PNG-encoded image bytes. + pub image_bytes: Vec, + /// True when every pixel has the same color (blank page after render). + pub is_solid_fill: bool, + /// Solid rectangles/lines detected in the raster (viewport coords). + /// Empty unless `detectScreenshotRects` is enabled. + pub rects: Vec, +} + +/// A solid rectangle or line detected in a rendered page. +#[derive(Serialize, Tsify)] +#[tsify(into_wasm_abi)] +#[serde(rename_all = "camelCase")] +pub struct ScreenshotRect { + pub x: f32, + pub y: f32, + pub width: f32, + pub height: f32, + /// Fill color as ARGB hex string (e.g. "ff1a2b3c"). + pub color: String, + /// True when the rect is thin enough to be a rule/divider line. + pub is_line: bool, +} + +#[derive(Serialize, Tsify)] +#[tsify(into_wasm_abi)] +#[serde(rename_all = "camelCase")] +pub struct DocumentMetadata { + #[serde(skip_serializing_if = "Option::is_none")] + pub creation_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mod_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_encrypted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub security_handler_revision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub permissions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub eof_section_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub startxref_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub trailer_id_pair_differs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_file_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub xmp: Option, + /// True when the catalog's XMP stream exceeded the 64 KiB cap. WASM + /// builds never populate `xmp`, so this is always absent there. + #[serde(skip_serializing_if = "Option::is_none")] + pub xmp_truncated: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature_byte_range_reaches_eof: Option, +} + +impl From<&liteparse::types::DocumentMetadata> for DocumentMetadata { + fn from(metadata: &liteparse::types::DocumentMetadata) -> Self { + Self { + creation_date: metadata.creation_date.clone(), + mod_date: metadata.mod_date.clone(), + file_version: metadata.file_version, + is_encrypted: metadata.is_encrypted, + security_handler_revision: metadata.security_handler_revision, + permissions: metadata.permissions.map(|value| value as f64), + eof_section_count: metadata.eof_section_count, + startxref_count: metadata.startxref_count, + trailer_id_pair_differs: metadata.trailer_id_pair_differs, + raw_file_size: metadata.raw_file_size.map(|value| value as f64), + xmp: metadata.xmp.clone(), + xmp_truncated: metadata.xmp_truncated, + signature_count: metadata.signature_count, + signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, + } + } +} + /// One raw packet from an XFA form document's `/XFA` array. #[derive(Serialize, Tsify)] #[tsify(into_wasm_abi)] @@ -841,166 +976,214 @@ impl LiteParse { .await .map_err(|e| JsError::new(&format!("parse failed: {}", e)))?; - let extract_text_metadata = self.inner.config().extract_text_metadata; - let pages: Vec = result - .pages - .iter() - .map(|p| ParsedPage { - page_num: p.page_number, - width: p.page_width, - height: p.page_height, - content_bounds: p.content_bounds.as_ref().map(|b| VectorRect { - x: b.x, - y: b.y, - width: b.width, - height: b.height, - }), - text: p.text.clone(), - markdown: p.markdown.clone(), - text_items: p - .text_items + Ok(to_js_result( + &result, + self.inner.config().extract_text_metadata, + )) + } +} + +/// Convert a core [`CoreParseResult`] into the wasm-bindgen view. +/// +/// Shared by `parse()` and `ParseSession::nextBatch()` so a batch is mapped +/// exactly the same way a whole-document result is. +fn to_js_result(result: &liteparse::ParseResult, extract_text_metadata: bool) -> ParseResult { + let pages: Vec = result + .pages + .iter() + .map(|p| ParsedPage { + page_num: p.page_number, + width: p.page_width, + height: p.page_height, + content_bounds: p.content_bounds.as_ref().map(|b| VectorRect { + x: b.x, + y: b.y, + width: b.width, + height: b.height, + }), + text: p.text.clone(), + markdown: p.markdown.clone(), + text_items: p + .text_items + .iter() + .map(|i| { + // Core-gated metadata view; `TextMetadata` defines + // which fields `extractTextMetadata` covers. + let meta = i.text_metadata(extract_text_metadata); + TextItem { + text: i.text.clone(), + x: i.x, + y: i.y, + width: i.width, + height: i.height, + font_name: i.font_name.clone(), + font_size: i.font_size, + confidence: i.confidence, + rotation: i.rotation, + font_height: meta.font_height, + font_ascent: meta.font_ascent, + font_descent: meta.font_descent, + font_weight: meta.font_weight, + text_width: meta.text_width, + font_is_buggy: meta.font_is_buggy, + mcid: meta.mcid, + fill_color: meta.fill_color.map(str::to_owned), + stroke_color: meta.stroke_color.map(str::to_owned), + char_codes: meta + .char_codes + .filter(|codes| !codes.is_empty()) + .map(<[u32]>::to_vec), + trailing_space_generated: meta.trailing_space_generated, + words: if i.words.is_empty() { + None + } else { + Some( + i.words + .iter() + .map(|w| WordBox { + text: w.text.clone(), + x: w.x, + y: w.y, + width: w.width, + height: w.height, + }) + .collect(), + ) + }, + } + }) + .collect(), + complexity: p.complexity.as_ref().map(PageComplexityStats::from_rust), + vector_graphics: p.vector_graphics.as_ref().map(|v| VectorGraphics { + shapes: v + .shapes .iter() - .map(|i| { - // Core-gated metadata view; `TextMetadata` defines - // which fields `extractTextMetadata` covers. - let meta = i.text_metadata(extract_text_metadata); - TextItem { - text: i.text.clone(), - x: i.x, - y: i.y, - width: i.width, - height: i.height, - font_name: i.font_name.clone(), - font_size: i.font_size, - confidence: i.confidence, - rotation: i.rotation, - font_height: meta.font_height, - font_ascent: meta.font_ascent, - font_descent: meta.font_descent, - font_weight: meta.font_weight, - text_width: meta.text_width, - font_is_buggy: meta.font_is_buggy, - mcid: meta.mcid, - fill_color: meta.fill_color.map(str::to_owned), - stroke_color: meta.stroke_color.map(str::to_owned), - char_codes: meta - .char_codes - .filter(|codes| !codes.is_empty()) - .map(<[u32]>::to_vec), - trailing_space_generated: meta.trailing_space_generated, - words: if i.words.is_empty() { - None - } else { - Some( - i.words - .iter() - .map(|w| WordBox { - text: w.text.clone(), - x: w.x, - y: w.y, - width: w.width, - height: w.height, - }) - .collect(), - ) - }, - } + .map(|s| VectorShape { + bbox: VectorRect { + x: s.bbox.x, + y: s.bbox.y, + width: s.bbox.width, + height: s.bbox.height, + }, + stroke: s.stroke, + stroke_color: s.stroke_color.clone(), + fill: s.fill, + fill_color: s.fill_color.clone(), + has_curve: s.has_curve, }) .collect(), - complexity: p.complexity.as_ref().map(PageComplexityStats::from_rust), - vector_graphics: p.vector_graphics.as_ref().map(|v| VectorGraphics { - shapes: v - .shapes - .iter() - .map(|s| VectorShape { - bbox: VectorRect { - x: s.bbox.x, - y: s.bbox.y, - width: s.bbox.width, - height: s.bbox.height, - }, - stroke: s.stroke, - stroke_color: s.stroke_color.clone(), - fill: s.fill, - fill_color: s.fill_color.clone(), - has_curve: s.has_curve, - }) - .collect(), - lines: v - .lines - .iter() - .map(|l| VectorLine { - x1: l.x1, - y1: l.y1, - x2: l.x2, - y2: l.y2, - stroke: l.stroke, - stroke_width: l.stroke_width, - stroke_color: l.stroke_color.clone(), - fill: l.fill, - fill_color: l.fill_color.clone(), - }) - .collect(), - }), - annotations: p.annotations.as_ref().map(|annotations| { - annotations - .iter() - .map(DocumentAnnotation::from_rust) - .collect() - }), - form_fields: p - .form_fields - .as_ref() - .map(|fields| fields.iter().map(FormField::from_rust).collect()), - structure_tree: p.structure_tree.as_ref().map(StructureTree::from_rust), - }) - .collect(); - - let images: Vec = result - .images - .iter() - .map(|img| ExtractedImage { - id: img.id.clone(), - name: img.name.clone(), - path: img.path.clone(), - page: img.page, - bbox: ImageRect { - x: img.bbox.x, - y: img.bbox.y, - width: img.bbox.width, - height: img.bbox.height, - }, - width: img.width, - height: img.height, - rotation: img.rotation, - format: img.format.clone(), - duplicate_of: img.duplicate_of.clone(), - bytes: img.bytes.as_slice().to_vec(), - }) - .collect(); - - Ok(ParseResult { - pages, - text: result.text.clone(), - images, - image_error_count: result.image_error_count, - form_type: result.form_type, - creator: result.creator.clone(), - producer: result.producer.clone(), - xfa_packets: result.xfa_packets.as_ref().map(|packets| { - packets + lines: v + .lines .iter() - .map(|packet| XfaPacket { - index: packet.index, - name: packet.name.clone(), - content_length: packet.content_length, - content: packet.content.clone(), + .map(|l| VectorLine { + x1: l.x1, + y1: l.y1, + x2: l.x2, + y2: l.y2, + stroke: l.stroke, + stroke_width: l.stroke_width, + stroke_color: l.stroke_color.clone(), + fill: l.fill, + fill_color: l.fill_color.clone(), }) + .collect(), + }), + annotations: p.annotations.as_ref().map(|annotations| { + annotations + .iter() + .map(DocumentAnnotation::from_rust) .collect() }), + form_fields: p + .form_fields + .as_ref() + .map(|fields| fields.iter().map(FormField::from_rust).collect()), + structure_tree: p.structure_tree.as_ref().map(StructureTree::from_rust), }) + .collect(); + + let images: Vec = result + .images + .iter() + .map(|img| ExtractedImage { + id: img.id.clone(), + name: img.name.clone(), + path: img.path.clone(), + page: img.page, + bbox: ImageRect { + x: img.bbox.x, + y: img.bbox.y, + width: img.bbox.width, + height: img.bbox.height, + }, + width: img.width, + height: img.height, + rotation: img.rotation, + format: img.format.clone(), + duplicate_of: img.duplicate_of.clone(), + bytes: img.bytes.as_slice().to_vec(), + }) + .collect(); + + let screenshots: Vec = result + .screenshots + .iter() + .map(|shot| ScreenshotResult { + page_num: shot.page_num, + width: shot.width, + height: shot.height, + image_bytes: shot.image_bytes.clone(), + is_solid_fill: shot.is_solid_fill, + rects: shot + .rects + .iter() + .map(|rect| ScreenshotRect { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + color: rect.color.clone(), + is_line: rect.is_line, + }) + .collect(), + }) + .collect(); + + ParseResult { + total_pages: result.total_pages, + pages, + text: result.text.clone(), + images, + screenshots, + image_error_count: result.image_error_count, + page_errors: result + .page_errors + .iter() + .map(|error| PageError { + page_num: error.page_number, + message: error.message.clone(), + }) + .collect(), + form_type: result.form_type, + creator: result.creator.clone(), + producer: result.producer.clone(), + doc_meta: result.doc_meta.as_ref().map(DocumentMetadata::from), + xfa_packets: result.xfa_packets.as_ref().map(|packets| { + packets + .iter() + .map(|packet| XfaPacket { + index: packet.index, + name: packet.name.clone(), + content_length: packet.content_length, + content: packet.content.clone(), + }) + .collect() + }), } +} +#[wasm_bindgen] +impl LiteParse { /// Determine per-page complexity for the given PDF bytes. Returns /// `Promise` — a cheap pre-OCR check with per-page /// signals and a `needsOcr` verdict. @@ -1014,6 +1197,88 @@ impl LiteParse { Ok(stats.iter().map(PageComplexityStats::from_rust).collect()) } + + /// Open PDF bytes for bounded-memory batch parsing. Returns + /// `Promise`. + /// + /// Yields `batchSize` pages at a time (default 25), which bounds both the + /// Rust-side result and the JS objects built from it — the wasm heap has a + /// hard ceiling, so a large document parsed whole can exhaust it. + /// + /// Cross-page passes (repeated header/footer removal, image deduplication) + /// see only the pages in their own batch, so output can differ from a + /// whole-document `parse()`. + #[wasm_bindgen(js_name = openBatchSession)] + pub async fn open_batch_session( + &self, + data: Vec, + batch_size: Option, + ) -> Result { + let batch_size = batch_size.unwrap_or(liteparse::DEFAULT_PAGE_BATCH_SIZE); + let session = self + .inner + .open_batch_session(PdfInput::Bytes(data), batch_size) + .await + .map_err(|e| JsError::new(&format!("open failed: {}", e)))?; + + Ok(ParseSession { + extract_text_metadata: self.inner.config().extract_text_metadata, + inner: session, + }) + } +} + +/// One batch of pages from a [`ParseSession`]. +#[derive(Serialize, Tsify)] +#[tsify(into_wasm_abi)] +#[serde(rename_all = "camelCase")] +pub struct ParseBatch { + /// First source page in this batch, 1-indexed. + pub start_page: u32, + /// Last source page in this batch, 1-indexed and inclusive. + pub end_page: u32, + /// The pages in `startPage..=endPage`, as an ordinary parse result. + pub result: ParseResult, +} + +/// A document opened once and parsed in bounded page batches. +/// +/// Created by `LiteParse.openBatchSession()`. Call `nextBatch()` until it +/// returns `undefined`, then `free()` the session to release its wasm-side +/// memory promptly (wasm-bindgen objects are not garbage collected). +#[wasm_bindgen] +pub struct ParseSession { + inner: liteparse::ParseSession, + extract_text_metadata: bool, +} + +#[wasm_bindgen] +impl ParseSession { + /// Total pages in the source document, before `maxPages` or batching. + #[wasm_bindgen(getter, js_name = totalPages)] + pub fn total_pages(&self) -> u32 { + self.inner.total_pages() + } + + /// Parse and return the next batch, or `undefined` once every page within + /// `maxPages` has been yielded. + /// + /// Batches are parsed one at a time: await each call before making the + /// next (a concurrent call throws wasm-bindgen's recursive-borrow error). + #[wasm_bindgen(js_name = nextBatch)] + pub async fn next_batch(&mut self) -> Result, JsError> { + let batch = self + .inner + .next_batch() + .await + .map_err(|e| JsError::new(&format!("batch parse failed: {}", e)))?; + + Ok(batch.map(|batch| ParseBatch { + start_page: batch.start_page, + end_page: batch.end_page, + result: to_js_result(&batch.result, self.extract_text_metadata), + })) + } } #[derive(Serialize, Tsify)] diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.toml index 081f90b9..ec237330 100644 --- a/crates/liteparse/Cargo.toml +++ b/crates/liteparse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "liteparse" -version = "2.10.1" +version = "2.12.0" edition.workspace = true license.workspace = true repository.workspace = true @@ -31,9 +31,10 @@ blake3 = "1" clap = { version = "4.5.55", features = ["derive"] } file-format = { version = "0.29.0", features = ["reader"] } image = { version = "0.25", default-features = false, features = ["default-formats"] } +memchr = "2" ordered-float = "5.3.0" -pdfium = { package = "liteparse-pdfium", version = "1.4.0", path = "../pdfium" } -pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.4.0", path = "../pdfium-sys" } +pdfium = { package = "liteparse-pdfium", version = "1.7.0", path = "../pdfium" } +pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.7.0", path = "../pdfium-sys" } reqwest = { version = "0.13.3", default-features = false, features = ["json", "form", "multipart", "rustls"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" diff --git a/crates/liteparse/src/bidi.rs b/crates/liteparse/src/bidi.rs new file mode 100644 index 00000000..a509a99f --- /dev/null +++ b/crates/liteparse/src/bidi.rs @@ -0,0 +1,135 @@ +//! Minimal bidirectional-text helpers. +//! +//! PDF content streams store right-to-left text in *logical* order already — +//! a correctly generated Hebrew/Arabic PDF places the first logical character +//! at the highest x and advances leftward, and PDFium hands the characters back +//! in that same stream order. So liteparse never needs to reverse characters; +//! what it does need is to stop assuming that "left to right on the page" means +//! "first to last in reading order". +//! +//! Two places care: +//! * `extract` measures inter-character gaps along the writing direction +//! (see `SegmentBuilder::gap_to`), so RTL runs are not sheared into +//! fragments by the line-change heuristics. +//! * line assembly concatenates a line's items in reading order, which for an +//! RTL line runs right-to-left across the page. + +/// Strong right-to-left character (Unicode bidi class R or AL). Covers Hebrew, +/// Arabic, Syriac, Thaana, N'Ko, Samaritan, Mandaic, Adlam and the Arabic +/// presentation-form blocks that subset fonts commonly map to. +pub(crate) fn is_rtl_char(c: char) -> bool { + matches!(c as u32, + 0x0590..=0x05FF // Hebrew + | 0x0600..=0x07BF // Arabic, Syriac, Thaana, N'Ko + | 0x0800..=0x085F // Samaritan, Mandaic + | 0x08A0..=0x08FF // Arabic Extended-A + | 0xFB1D..=0xFDFF // Hebrew/Arabic presentation forms A + | 0xFE70..=0xFEFF // Arabic presentation forms B + | 0x10800..=0x10FFF // Cypriot..Old Hungarian etc. + | 0x1E800..=0x1EFFF // Mende Kikakui, Adlam, Arabic Math + ) +} + +/// Strong left-to-right character. Deliberately *not* the complement of +/// [`is_rtl_char`]: digits, punctuation and whitespace are bidi-neutral and +/// must not vote on a line's base direction, or every "12.34 SAR" amount on an +/// Arabic invoice would drag its line back to LTR. +fn is_strong_ltr_char(c: char) -> bool { + c.is_alphabetic() && !is_rtl_char(c) +} + +/// Base direction of a run of characters, decided by majority of +/// strong-direction characters. Neutral-only text (pure numbers, punctuation) +/// is LTR, which keeps every left-to-right document on exactly the path it took +/// before. +fn is_rtl_chars(chars: impl Iterator) -> bool { + let mut rtl = 0usize; + let mut ltr = 0usize; + for c in chars { + if is_rtl_char(c) { + rtl += 1; + } else if is_strong_ltr_char(c) { + ltr += 1; + } + } + rtl > ltr +} + +/// Base direction of a line of text. See [`is_rtl_chars`]. +pub(crate) fn is_rtl_text(s: &str) -> bool { + !s.is_ascii() && is_rtl_chars(s.chars()) +} + +/// Base direction of a line still held as separate pieces, without joining +/// them first — line assembly needs the direction *before* it picks a join +/// order, and the pieces can be numerous. +/// +/// Every RTL character is non-ASCII, so a line whose pieces are all ASCII has +/// an RTL count of zero and cannot be RTL whatever its LTR count. `is_ascii` is +/// a byte-wise scan rather than per-`char` classification, which keeps this off +/// the hot path for left-to-right documents — the overwhelmingly common case. +pub(crate) fn is_rtl_pieces<'a>(pieces: impl IntoIterator + Clone) -> bool { + if pieces.clone().into_iter().all(|p| p.is_ascii()) { + return false; + } + is_rtl_chars(pieces.into_iter().flat_map(str::chars)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strong_direction_classification() { + assert!(is_rtl_char('\u{05DE}')); // Hebrew mem + assert!(is_rtl_char('\u{0642}')); // Arabic qaf + assert!(!is_rtl_char('A')); + assert!(!is_rtl_char('7')); + // Digits and punctuation are neutral, not strong LTR. + assert!(!is_strong_ltr_char('7')); + assert!(!is_strong_ltr_char('.')); + assert!(is_strong_ltr_char('A')); + } + + #[test] + fn line_direction_ignores_neutrals() { + // An Arabic label with a Latin currency code and an amount stays RTL: + // the digits must not outvote the letters. + assert!(is_rtl_text("المجموع الفرعي: 75.00")); + assert!(is_rtl_text("סך הכל 12.34")); + // A Latin line with a stray Hebrew word stays LTR. + assert!(!is_rtl_text("Total due 87.75 ILS")); + // Neutral-only lines stay LTR so LTR documents are untouched. + assert!(!is_rtl_text("2026-07-27 10:30")); + assert!(!is_rtl_text("")); + } + + #[test] + fn ascii_fast_path_agrees_with_full_scan() { + // The `is_ascii` short-circuit must be an optimization only: for any + // input it has to return exactly what the per-char scan would. + for s in [ + "", + "Total due 87.75 ILS", + "2026-07-27", + "!@#$%^&*()", + "\u{05DE}\u{05E1}", + "mixed \u{0642} latin", + "caf\u{e9} na\u{ef}ve", + ] { + assert_eq!(is_rtl_text(s), is_rtl_chars(s.chars()), "mismatch on {s:?}"); + assert_eq!(is_rtl_pieces([s]), is_rtl_chars(s.chars())); + } + // Split across pieces, where no single piece is decisive. + assert_eq!( + is_rtl_pieces(["12.34 ", "\u{05DE}\u{05E1}\u{05E2}"]), + is_rtl_chars("12.34 \u{05DE}\u{05E1}\u{05E2}".chars()) + ); + } + + #[test] + fn currency_code_does_not_flip_short_rtl_line() { + // "USD" is 3 strong LTR chars; the Hebrew must still win. + assert!(is_rtl_text("סך הכל לתשלום 87.75 ILS")); + } +} diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rs index bffa6012..a4bca65c 100644 --- a/crates/liteparse/src/config.rs +++ b/crates/liteparse/src/config.rs @@ -19,6 +19,16 @@ pub struct LiteParseConfig { pub max_pages: usize, /// Specific pages to parse (e.g., "1-5,10,15-20"). None means all pages. pub target_pages: Option, + /// Render parsed pages to PNG and return them in `ParseResult.screenshots`. + /// Default `false`; PNG payloads can be large. + #[serde(default)] + pub extract_screenshots: bool, + /// Continue parsing after a page-level PDFium extraction failure and + /// report it in `ParseResult.page_errors`. Default `false` preserves the + /// fail-fast behavior. Document-open and document-level failures remain + /// fatal regardless of this setting. + #[serde(default)] + pub continue_on_page_error: bool, /// DPI for rendering pages (used for OCR and screenshots). pub dpi: f32, /// Output format. @@ -75,6 +85,11 @@ pub struct LiteParseConfig { /// yield an empty list. #[serde(default)] pub extract_xfa_packets: bool, + /// Collect document-level provenance metadata into `ParseResult.doc_meta`. + /// Default `false`: `None` for inputs converted from a non-PDF format, + /// since the facts would describe the intermediate PDF rather than the caller's file. + #[serde(default)] + pub extract_document_metadata: bool, /// Detect solid rectangles and thick lines in rendered page screenshots /// and attach them to each `ScreenshotResult.rects`. Works on the raster, /// so it also finds structure in scanned/flattened pages that have no @@ -208,6 +223,8 @@ impl Default for LiteParseConfig { tessdata_path: None, max_pages: 1000, target_pages: None, + extract_screenshots: false, + continue_on_page_error: false, dpi: 150.0, output_format: OutputFormat::Json, preserve_very_small_text: false, @@ -224,6 +241,7 @@ impl Default for LiteParseConfig { extract_structure_tree: false, extract_content_bounds: false, extract_xfa_packets: false, + extract_document_metadata: false, detect_screenshot_rects: false, render_form_fields: false, ocr_failure_fatal: true, @@ -253,6 +271,13 @@ fn default_num_workers() -> usize { /// input. const MAX_TARGET_PAGES: u64 = 100_000; +/// Pages per batch when a caller of +/// [`crate::parser::LiteParse::open_batch_session`] does +/// not pick a size. Small enough to keep the materialized result bounded; +/// large enough that the per-batch document reopen stays modest (it costs +/// roughly 13% at this size on a 457-page document, ~4% at 50). +pub const DEFAULT_PAGE_BATCH_SIZE: usize = 25; + #[doc(hidden)] pub fn parse_target_pages(s: &str) -> Result, String> { let mut pages = Vec::new(); @@ -347,6 +372,8 @@ mod tests { // OCR defaults on only when a built-in engine is compiled in. assert_eq!(c.ocr_enabled, cfg!(feature = "tesseract")); assert_eq!(c.max_pages, 1000); + assert!(!c.extract_screenshots); + assert!(!c.continue_on_page_error); assert_eq!(c.dpi, 150.0); assert_eq!(c.output_format, OutputFormat::Json); assert!(!c.preserve_very_small_text); diff --git a/crates/liteparse/src/conversion.rs b/crates/liteparse/src/conversion.rs index 8bb407fb..dc45223b 100644 --- a/crates/liteparse/src/conversion.rs +++ b/crates/liteparse/src/conversion.rs @@ -88,6 +88,15 @@ pub struct PdfInputGuard { temps: Vec, } +impl PdfInputGuard { + /// True when the resolved input is a temporary PDF produced by converting + /// a non-PDF source, so raw-file facts describe the intermediate, not the + /// document the caller passed in. + pub fn is_converted(&self) -> bool { + !self.temps.is_empty() + } +} + /// Resolve a document input to a PDF suitable for rendering or text extraction. /// /// When `reject_text_formats` is true, plain-text files (`.txt`, etc.) return a diff --git a/crates/liteparse/src/document_metadata.rs b/crates/liteparse/src/document_metadata.rs new file mode 100644 index 00000000..c28ad7b5 --- /dev/null +++ b/crates/liteparse/src/document_metadata.rs @@ -0,0 +1,257 @@ +//! Document-level PDF provenance metadata extraction. + +use crate::types::{DocumentMetadata, PdfInput}; +use pdfium::Document; +use std::io::{Read, Seek, SeekFrom}; + +const SCAN_BUFFER_BYTES: usize = 1 << 20; +const SCAN_OVERLAP: usize = 64; +const XMP_MAX_BYTES: usize = 64 * 1024; +/// Above this size, resolving the catalog's `/Metadata` object costs more than +/// the whole parse: lopdf decodes every stream, so a 103 MB file takes ~7.8 s +/// (worst case measured under the cap is ~0.5 s). Larger documents report no +/// `xmp` at all rather than a cheaper guess — the first `) -> DocumentMetadata { + let mut metadata = match input { + #[cfg(not(target_arch = "wasm32"))] + PdfInput::Path(path) => std::fs::File::open(path) + .ok() + .map(|mut file| extract_raw_facts(&mut file)) + .unwrap_or_default(), + PdfInput::Bytes(bytes) => extract_raw_facts(&mut std::io::Cursor::new(bytes)), + #[cfg(target_arch = "wasm32")] + PdfInput::Path(_) => DocumentMetadata::default(), + }; + + metadata.creation_date = document.meta_text("CreationDate"); + metadata.mod_date = document.meta_text("ModDate"); + metadata.file_version = document.file_version(); + let security_revision = document.security_handler_revision(); + metadata.is_encrypted = Some(security_revision != -1); + if security_revision != -1 { + metadata.security_handler_revision = Some(security_revision); + metadata.permissions = Some(document.permissions()); + } + let signatures = document.signature_summary(metadata.raw_file_size); + metadata.signature_count = signatures.count; + metadata.signature_byte_range_reaches_eof = signatures.byte_range_reaches_eof; + + #[cfg(not(target_arch = "wasm32"))] + if let Some((xmp, truncated)) = catalog_xmp(input, metadata.raw_file_size) { + metadata.xmp = Some(xmp); + metadata.xmp_truncated = Some(truncated); + } + metadata +} + +/// Read the document catalog's `/Metadata` XMP stream — the only XMP that is +/// certainly the document's own. `None` when the file is too large to parse +/// cheaply, has no catalog metadata, or cannot be decoded (encrypted or +/// damaged); the caller then reports no XMP. +#[cfg(not(target_arch = "wasm32"))] +fn catalog_xmp(input: &PdfInput, file_size: Option) -> Option<(String, bool)> { + if file_size? > XMP_CATALOG_MAX_FILE_BYTES { + return None; + } + let document = match input { + PdfInput::Path(path) => lopdf::Document::load(path).ok()?, + PdfInput::Bytes(bytes) => lopdf::Document::load_mem(bytes).ok()?, + }; + let object = document.catalog().ok()?.get(b"Metadata").ok()?; + let stream = document.dereference(object).ok()?.1.as_stream().ok()?; + let bytes = stream + .decompressed_content() + .unwrap_or_else(|_| stream.content.clone()); + let truncated = bytes.len() > XMP_MAX_BYTES; + let text = String::from_utf8_lossy(&bytes[..bytes.len().min(XMP_MAX_BYTES)]).into_owned(); + (!text.trim().is_empty()).then_some((text, truncated)) +} + +fn extract_raw_facts(reader: &mut R) -> DocumentMetadata { + let mut metadata = DocumentMetadata::default(); + let file_size = reader.seek(SeekFrom::End(0)).ok(); + metadata.raw_file_size = file_size; + if reader.seek(SeekFrom::Start(0)).is_err() { + return metadata; + } + + let mut buffer = vec![0u8; SCAN_BUFFER_BYTES]; + let mut carry = 0usize; + let mut eof_count = 0u32; + let mut startxref_count = 0u32; + + loop { + // Must be a full fill, not a single `read`: a short read would shrink + // the window below a marker's length and lose it entirely. + let got = fill_buffer(reader, &mut buffer[carry..]); + let window_len = carry + got; + if window_len == 0 { + break; + } + let countable = if got > 0 && window_len > SCAN_OVERLAP { + window_len - SCAN_OVERLAP + } else { + window_len + }; + let window = &buffer[..window_len]; + eof_count = eof_count.saturating_add(count_occurrences_before(window, b"%%EOF", countable)); + startxref_count = startxref_count.saturating_add(count_occurrences_before( + window, + b"startxref", + countable, + )); + if got == 0 { + break; + } + carry = window_len - countable; + buffer.copy_within(countable..window_len, 0); + } + + metadata.eof_section_count = Some(eof_count); + metadata.startxref_count = Some(startxref_count); + + if let Some(file_size) = file_size { + let tail_start = file_size.saturating_sub(SCAN_BUFFER_BYTES as u64); + if reader.seek(SeekFrom::Start(tail_start)).is_ok() + && let Some(tail) = read_up_to(reader, (file_size - tail_start) as usize) + { + metadata.trailer_id_pair_differs = trailer_id_pair_differs(&tail); + } + } + + metadata +} + +/// Read up to `max` bytes, retrying short reads. `None` when the reader +/// yielded nothing at all. +fn read_up_to(reader: &mut R, max: usize) -> Option> { + let mut buffer = vec![0u8; max]; + let got = fill_buffer(reader, &mut buffer); + buffer.truncate(got); + (got > 0).then_some(buffer) +} + +/// Fill `buf` completely, looping over short reads. Returns the byte count, +/// short only at EOF or on a read error. +fn fill_buffer(reader: &mut R, buf: &mut [u8]) -> usize { + let mut filled = 0; + while filled < buf.len() { + match reader.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(got) => filled += got, + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(_) => break, + } + } + filled +} + +fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { + memchr::memmem::find(haystack, needle) +} + +fn count_occurrences_before(haystack: &[u8], needle: &[u8], start_limit: usize) -> u32 { + memchr::memmem::find_iter(haystack, needle) + .take_while(|offset| *offset < start_limit) + .count() + .min(u32::MAX as usize) as u32 +} + +fn trailer_id_pair_differs(bytes: &[u8]) -> Option { + let mut cursor = 0usize; + let mut last_pair: Option<(&[u8], &[u8])> = None; + while let Some(offset) = find_bytes(&bytes[cursor..], b"/ID") { + let id_start = cursor + offset; + let mut pos = id_start + 3; + while bytes + .get(pos) + .is_some_and(|b| matches!(b, b' ' | b'\r' | b'\n' | b'\t')) + { + pos += 1; + } + if bytes.get(pos) == Some(&b'[') { + pos += 1; + let mut values = Vec::with_capacity(2); + while pos < bytes.len() && bytes[pos] != b']' && values.len() < 2 { + if bytes[pos] == b'<' { + let start = pos + 1; + if let Some(close) = bytes[start..].iter().position(|b| *b == b'>') { + values.push(&bytes[start..start + close]); + pos = start + close; + } + } + pos += 1; + } + if values.len() == 2 { + last_pair = Some((values[0], values[1])); + } + } + cursor = id_start + 3; + } + last_pair.map(|(first, second)| first != second) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_raw_provenance_facts() { + let pdf = b"%PDF-1.4\n/ID []\nstartxref\n1\n%%EOF\n\ + update\n/ID [ ]\nstartxref\n2\n%%EOF\ntail"; + let metadata = extract_raw_facts(&mut std::io::Cursor::new(pdf)); + assert_eq!(metadata.raw_file_size, Some(pdf.len() as u64)); + assert_eq!(metadata.eof_section_count, Some(2)); + assert_eq!(metadata.startxref_count, Some(2)); + assert_eq!(metadata.trailer_id_pair_differs, Some(true)); + // XMP comes from the catalog only; the raw scan never guesses at it. + assert_eq!(metadata.xmp, None); + } + + /// A reader that hands back one byte at a time, like a slow pipe. + struct DripReader(std::io::Cursor>); + + impl Read for DripReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let take = buf.len().min(1); + self.0.read(&mut buf[..take]) + } + } + + impl Seek for DripReader { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.0.seek(pos) + } + } + + #[test] + fn short_reads_do_not_lose_markers_or_the_trailer() { + let pdf = b"%PDF-1.4\nbody\n/ID []\nstartxref\n1\n%%EOF\n"; + let metadata = extract_raw_facts(&mut DripReader(std::io::Cursor::new(pdf.to_vec()))); + assert_eq!(metadata.trailer_id_pair_differs, Some(true)); + assert_eq!(metadata.startxref_count, Some(1)); + assert_eq!(metadata.eof_section_count, Some(1)); + } + + #[test] + fn trailer_id_uses_last_valid_pair() { + assert_eq!( + trailer_id_pair_differs(b"/ID [] junk /ID []"), + Some(false) + ); + assert_eq!(trailer_id_pair_differs(b"no trailer id"), None); + } + + #[test] + fn finds_markers_that_cross_scan_chunks_without_double_counting() { + let mut pdf = vec![b'x'; SCAN_BUFFER_BYTES - 7]; + pdf.extend_from_slice(b"startxref\n%%EOF\npayload"); + let metadata = extract_raw_facts(&mut std::io::Cursor::new(pdf)); + assert_eq!(metadata.startxref_count, Some(1)); + assert_eq!(metadata.eof_section_count, Some(1)); + } +} diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs index e67d39ce..c5ae8fc0 100644 --- a/crates/liteparse/src/extract.rs +++ b/crates/liteparse/src/extract.rs @@ -1,13 +1,16 @@ +use crate::bidi::is_rtl_char; use crate::error::LiteParseError; use crate::glyph_names::resolve_glyph_name; use crate::types::{ DocumentAnnotation, ExtractedImage, FormField, GraphicPrimitive, ImageRef, OutlineTarget, - Page as LitePage, PdfInput, Rect, StructNode, StructureAttributeValue, StructureTree, - StructureTreeElement, TextItem, VectorGraphics, VectorLine, VectorShape, WordBox, + Page as LitePage, PageError, PdfInput, Rect, StructNode, StructureAttributeValue, + StructureTree, StructureTreeElement, TextItem, VectorGraphics, VectorLine, VectorShape, + WordBox, }; use image::ImageEncoder; use pdfium::{ - Document, Font, FontType, Library, Page, PathObject, PdfLink, RectF, SegmentKind, TextPage, + Document, Font, FontType, FormEnvironment, Library, Page, PathObject, PdfLink, RectF, + SegmentKind, TextPage, }; /// Open a PDF from path or bytes with an optional password. @@ -58,14 +61,26 @@ pub(crate) fn extract_pages_from_document( None, ExtractionOutputOptions::default(), )? - .0) + .pages) +} + +/// Output of [`extract_pages_and_images`]. +pub(crate) struct ExtractedPages { + pub pages: Vec, + pub page_errors: Vec, + /// Empty unless `output_options.extract_images` was set. + pub images: Vec, + pub image_error_count: u32, + /// Whether any page was flattened to recover form-widget text. Flattening + /// mutates the open PDFium document, so a caller that still needs the + /// original widget annotations must reopen the input. + pub flattened_form_widgets: bool, } /// Same as `extract_pages_from_document` but optionally also renders every /// raster image object to bytes (when `output_options.extract_images` is true). Returned /// `ExtractedImage`s carry the same ids the markdown emitter will reference, -/// so callers can match them up by id. When image extraction is disabled the -/// returned image vec is always empty. +/// so callers can match them up by id. pub(crate) fn extract_pages_and_images( document: &Document, target_pages: Option<&[u32]>, @@ -73,12 +88,17 @@ pub(crate) fn extract_pages_and_images( extract_links: bool, glyph_resolver: Option<&dyn crate::GlyphResolver>, output_options: ExtractionOutputOptions, -) -> Result<(Vec, Vec, u32), LiteParseError> { +) -> Result { let page_count = document.page_count(); let mut pages = Vec::new(); + let mut page_errors = Vec::new(); let mut images: Vec = Vec::new(); let mut image_cache = ImageCache::default(); let mut image_error_count = 0u32; + let mut flattened_form_widgets = false; + // One FFI call keeps the per-page annotation walk off the hot path for + // every document without an AcroForm catalog, which is nearly all of them. + let document_has_form = document.form_type() != 0; let form_environment = output_options .extract_form_fields .then(|| document.form_environment()) @@ -93,111 +113,234 @@ pub(crate) fn extract_pages_and_images( continue; } - if pages.len() >= max_pages { + if pages.len() + page_errors.len() >= max_pages { break; } - let page = document.page(page_index)?; + let page_result = extract_single_page( + document, + page_index, + page_number, + extract_links, + glyph_resolver, + &output_options, + form_environment.as_ref(), + document_has_form, + &mut image_cache, + ); + + match resolve_page_result( + page_number, + page_result, + output_options.continue_on_page_error, + &mut page_errors, + )? { + Some(extraction) => { + pages.push(extraction.page); + images.extend(extraction.images); + image_error_count += extraction.image_error_count; + flattened_form_widgets |= extraction.flattened_form_widgets; + } + // A failed page's cached renders must not seed dedup for later + // pages: a hit would emit `duplicate_of` pointing at an image id + // that never made it into the output. + None => image_cache.remove_page(page_number), + } + } + + Ok(ExtractedPages { + pages, + page_errors, + images, + image_error_count, + flattened_form_widgets, + }) +} + +/// Everything one successfully extracted page contributes to the document +/// output. Accumulated page-locally so a failed page is rolled back by +/// dropping this value (plus [`ImageCache::remove_page`] for its cache +/// inserts) instead of undoing shared-state mutations. +struct PageExtraction { + page: LitePage, + /// Rendered image bytes; empty unless `extract_images` was set. + images: Vec, + image_error_count: u32, + /// Whether this page was flattened to recover form-widget text. + flattened_form_widgets: bool, +} + +#[allow(clippy::too_many_arguments)] +fn extract_single_page( + document: &Document, + page_index: i32, + page_number: u32, + extract_links: bool, + glyph_resolver: Option<&dyn crate::GlyphResolver>, + output_options: &ExtractionOutputOptions, + form_environment: Option<&FormEnvironment<'_, '_>>, + document_has_form: bool, + image_cache: &mut ImageCache, +) -> Result { + let mut images = Vec::new(); + let mut image_error_count = 0u32; + let mut flattened_form_widgets = false; + let page = document.page(page_index)?; + let raw_page_width = page.width(); + let raw_page_height = page.height(); + let view_box = page.view_box().unwrap_or(RectF { + left: 0.0, + top: raw_page_height, + right: raw_page_width, + bottom: 0.0, + }); + // All extracted geometry is converted to the rotation-adjusted + // viewport coordinate space. Keep the page dimensions in that same + // space so projection, filtering, and consumers do not clip content + // at the unrotated MediaBox width on /Rotate 90 or /Rotate 270 pages. + let (page_width, page_height) = page.viewport_size(&view_box); + // Once a qualifying widget is found, PDFium flattens every visible + // annotation on the page. Collect every annotation-backed output first. + let links = if extract_links { + page.links(&view_box) + } else { + Vec::new() + }; + // Computed when emitted (`extract_content_bounds`) or needed + // internally by the white-fill heuristic (`extract_vector_graphics`). + let content_bounds = (output_options.extract_content_bounds + || output_options.extract_vector_graphics) + .then(|| { + page.content_bounds() + .map(|bounds| rect_from_pdfium(page.bounds_to_viewport(&view_box, &bounds))) + }) + .flatten(); + let paths = page.path_objects(&view_box); + let graphics = extract_layout_graphics(&paths); + let vector_graphics = output_options + .extract_vector_graphics + .then(|| build_vector_graphics(&paths, content_bounds.as_ref())); + let struct_nodes = extract_page_struct_nodes(&page, &view_box); + let extracted_refs = extract_page_image_refs(&page, page_number, output_options.extract_images); + let mut image_refs = extracted_refs.refs; + image_error_count += extracted_refs.error_count; + let pdf_annotations = (output_options.extract_annotations + || output_options.extract_structure_tree) + .then(|| page.annotations(&view_box)) + .unwrap_or_default(); + let annotations = output_options + .extract_annotations + .then(|| pdf_annotations.iter().map(document_annotation).collect()); + let structure_tree = output_options.extract_structure_tree.then(|| { + let annotations_by_object = pdf_annotations + .iter() + .filter(|annotation| annotation.subtype == "link") + .filter_map(|annotation| annotation.object_number.map(|n| (n, annotation))) + .collect::>(); + StructureTree { + roots: page + .structure_tree() + .into_iter() + .map(|element| structure_tree_element(element, &annotations_by_object)) + .collect(), + } + }); + let form_fields = output_options.extract_form_fields.then(|| { + form_environment.map_or_else(Vec::new, |form| { + page.form_fields(form, &view_box, page_number) + .into_iter() + .map(|field| FormField { + id: field.id, + field_type: field.field_type, + page: field.page, + annotation_index: field.annotation_index, + widget_index: field.widget_index, + object_number: field.object_number, + name: field.name, + alternate_name: field.alternate_name, + value: field.value, + export_value: field.export_value, + field_flags: field.field_flags, + control_count: field.control_count, + control_index: field.control_index, + checked: field.checked, + rect: field.rect.map(rect_from_pdfium), + options: field.options, + selected_options: field.selected_options, + }) + .collect() + }) + }); + + if output_options.extract_images && !image_refs.is_empty() { + let rendered = render_page_images(&page, page_number, &image_refs, image_cache); + image_error_count += rendered.error_count; + images.extend(rendered.images); + for image_ref in &mut image_refs { + image_ref.jpeg_bytes = None; + image_ref.raw_bytes = None; + } + } + + // PDFium's text API reads only the page content stream. Filled form + // values commonly live in widget appearance streams, so promote only + // those widget appearances into page content and reload before text + // extraction. Non-widget annotations are excluded, and this does not + // initialize the form environment or execute document JS. + // + // `widget_text_rects` is empty for the overwhelming majority of pages — + // documents with no AcroForm catalog never even reach the annotation + // walk — so the whole path costs one `form_type()` call for most files. + let extract_text = |page: &Page| -> Result, LiteParseError> { let text_page = page.text()?; - let view_box = page.view_box().unwrap_or(RectF { - left: 0.0, - top: page.height(), - right: page.width(), - bottom: 0.0, - }); - let mut text_items = extract_page_text_items( - &page, + extract_page_text_items( + page, &text_page, &view_box, glyph_resolver, output_options.emit_word_boxes, output_options.extract_text_metadata, - )?; - if extract_links { - assign_links(&mut text_items, &page.links(&view_box)); - } - // Computed when emitted (`extract_content_bounds`) or needed - // internally by the white-fill heuristic (`extract_vector_graphics`). - let content_bounds = (output_options.extract_content_bounds - || output_options.extract_vector_graphics) - .then(|| { - page.content_bounds() - .map(|bounds| rect_from_pdfium(page.bounds_to_viewport(&view_box, &bounds))) - }) - .flatten(); - let paths = page.path_objects(&view_box); - let graphics = extract_layout_graphics(&paths); - let vector_graphics = output_options - .extract_vector_graphics - .then(|| build_vector_graphics(&paths, content_bounds.as_ref())); - assign_strikethrough(&mut text_items, &graphics); - let struct_nodes = extract_page_struct_nodes(&page, &view_box); - let extracted_refs = - extract_page_image_refs(&page, page_number, output_options.extract_images); - let mut image_refs = extracted_refs.refs; - image_error_count += extracted_refs.error_count; - let pdf_annotations = (output_options.extract_annotations - || output_options.extract_structure_tree) - .then(|| page.annotations(&view_box)) - .unwrap_or_default(); - let annotations = output_options - .extract_annotations - .then(|| pdf_annotations.iter().map(document_annotation).collect()); - let structure_tree = output_options.extract_structure_tree.then(|| { - let annotations_by_object = pdf_annotations - .iter() - .filter(|annotation| annotation.subtype == "link") - .filter_map(|annotation| annotation.object_number.map(|n| (n, annotation))) - .collect::>(); - StructureTree { - roots: page - .structure_tree() - .into_iter() - .map(|element| structure_tree_element(element, &annotations_by_object)) - .collect(), - } - }); - let form_fields = output_options.extract_form_fields.then(|| { - form_environment.as_ref().map_or_else(Vec::new, |form| { - page.form_fields(form, &view_box, page_number) - .into_iter() - .map(|field| FormField { - id: field.id, - field_type: field.field_type, - page: field.page, - annotation_index: field.annotation_index, - widget_index: field.widget_index, - object_number: field.object_number, - name: field.name, - alternate_name: field.alternate_name, - value: field.value, - export_value: field.export_value, - field_flags: field.field_flags, - control_count: field.control_count, - control_index: field.control_index, - checked: field.checked, - rect: field.rect.map(rect_from_pdfium), - options: field.options, - selected_options: field.selected_options, - }) - .collect() - }) - }); - - if output_options.extract_images && !image_refs.is_empty() { - let rendered = render_page_images(&page, page_number, &image_refs, &mut image_cache); - image_error_count += rendered.error_count; - images.extend(rendered.images); - for image_ref in &mut image_refs { - image_ref.jpeg_bytes = None; - image_ref.raw_bytes = None; + ) + }; + let widget_text_rects = if document_has_form { + page.form_widget_text_rects(&view_box) + } else { + Vec::new() + }; + let mut text_items = if widget_text_rects.is_empty() { + extract_text(&page)? + } else { + // PDFium's text layer keeps only one of two runs that start at + // essentially the same point, so a flattened appearance can + // suppress page text it lands on. Usually widget rects sit over + // blank space, and this bounds-only probe says so without touching + // the text API; only when a widget really does cover existing text + // do we extract twice and put back what was suppressed. + let overlaps_existing_text = page.text_objects_overlap(&view_box, &widget_text_rects); + let before = overlaps_existing_text + .then(|| extract_text(&page)) + .transpose()?; + drop(page); + match document.flatten_form_widgets(page_index)? { + Some(flattened_page) => { + flattened_form_widgets = true; + let mut items = extract_text(&flattened_page)?; + if let Some(before) = before { + restore_flattened_over_text(&mut items, before, &widget_text_rects); + } + items } + None => extract_text(&document.page(page_index)?)?, } + }; + assign_links(&mut text_items, &links); + assign_strikethrough(&mut text_items, &graphics); - pages.push(LitePage { + Ok(PageExtraction { + page: LitePage { page_number: page_number as usize, - page_width: page.width(), - page_height: page.height(), + page_width, + page_height, content_bounds: output_options .extract_content_bounds .then_some(content_bounds) @@ -210,14 +353,83 @@ pub(crate) fn extract_pages_and_images( annotations, form_fields, structure_tree, - }); + }, + images, + image_error_count, + flattened_form_widgets, + }) +} + +fn resolve_page_result( + page_number: u32, + result: Result, + continue_on_page_error: bool, + page_errors: &mut Vec, +) -> Result, LiteParseError> { + match result { + Ok(value) => Ok(Some(value)), + Err(error) if continue_on_page_error => { + page_errors.push(PageError { + page_number, + message: error.to_string(), + }); + Ok(None) + } + Err(error) => Err(error), } +} - Ok((pages, images, image_error_count)) +/// Put back page text that flattening suppressed. +/// +/// PDFium's text layer emits only one of two text runs that start at +/// essentially the same point, so a flattened widget appearance can knock out +/// page text it lands on. When the two carry the *same* string — a producer +/// that wrote the value into both the content stream and the appearance — that +/// is precisely the dedup a partially flattened file needs, and matching on +/// trimmed text leaves it alone. When they differ, such as a pre-printed label +/// sitting where the value is typed, dropping one is pure data loss, so the +/// pre-flatten copy is restored. +/// +/// Only called for the page where a widget rect actually covers existing text; +/// `before` is the pre-flatten extraction of the same page. +/// +/// Note this recovers one direction only. If the collision goes the other way +/// PDFium can suppress the *appearance* text instead, and the field value is +/// lost with no pre-flatten copy to restore it from. +fn restore_flattened_over_text( + items: &mut Vec, + before: Vec, + widget_rects: &[RectF], +) { + let surviving: std::collections::HashSet<&str> = items + .iter() + .map(|item| item.text.trim()) + .filter(|text| !text.is_empty()) + .collect(); + let mut restored: Vec = before + .iter() + .filter(|item| { + let text = item.text.trim(); + !text.is_empty() + && !surviving.contains(text) + && widget_rects + .iter() + .any(|rect| rect_contains_center(rect, item)) + }) + .cloned() + .collect(); + items.append(&mut restored); +} + +fn rect_contains_center(rect: &RectF, item: &TextItem) -> bool { + let cx = item.x + item.width / 2.0; + let cy = item.y + item.height / 2.0; + cx >= rect.left && cx <= rect.right && cy >= rect.top && cy <= rect.bottom } #[derive(Debug, Clone, Copy, Default)] pub(crate) struct ExtractionOutputOptions { + pub continue_on_page_error: bool, pub extract_content_bounds: bool, pub extract_text_metadata: bool, pub extract_images: bool, @@ -521,6 +733,9 @@ const IMAGE_MAX_COVERAGE: f32 = 0.9; struct CachedImage { raw_bytes: Vec, id: String, + /// Source page of the canonical render; lets a failed page's inserts be + /// rolled back so later duplicates never reference a dropped image id. + page: u32, format: String, bytes: std::sync::Arc>, } @@ -569,6 +784,16 @@ impl ImageCache { .or_default() .push(entry); } + + /// Drop every entry rendered from `page_number`. Called when that page + /// fails after its images were cached, so a later duplicate can't resolve + /// to a canonical image that was rolled back out of the output. + fn remove_page(&mut self, page_number: u32) { + self.entries.retain(|_, bucket| { + bucket.retain(|entry| entry.page != page_number); + !bucket.is_empty() + }); + } } pub(crate) struct RenderedImages { @@ -650,6 +875,7 @@ fn render_page_images( CachedImage { raw_bytes, id: r.id.clone(), + page: page_number, format, bytes, }, @@ -662,9 +888,8 @@ fn render_page_images( } } -/// Encode RGBA pixel bytes to PNG. Lives here (always-compiled) rather than in -/// `render` so the image-embed path is available on wasm, where the `render` -/// module (page rasterization / screenshots) is compiled out. +/// Encode RGBA pixel bytes to PNG. Used by both the image-embed path and the +/// `render` module (page rasterization / screenshots). pub(crate) fn encode_png(rgba: &[u8], width: u32, height: u32) -> Result, LiteParseError> { let mut png_buf = Vec::new(); let encoder = image::codecs::png::PngEncoder::new(&mut png_buf); @@ -1341,22 +1566,26 @@ fn extract_page_text_items( let y_overlap = vp_loose.top < seg.vp_bottom + y_tolerance && vp_loose.bottom > seg.vp_top - y_tolerance; - let gap = vp_strict.left - seg.last_char_right; + // Gaps are measured along the segment's writing direction, so a + // right-to-left run reads exactly like a left-to-right one: positive + // means "further along the line", negative means "doubled back". + let gap = seg.gap_to(&vp_strict, c); // Detect line change using complementary checks: // 1. Strict vertical separation: char's strict top is well below last char's strict bottom - // 2. Line wrap: char goes back leftward AND strict top is below last char's strict bottom - // (even slightly), indicating text wrapped to a new line within the same text object - // 3. Very large leftward jump: if the char jumps back by more than the current + // 2. Line wrap: char goes back against the writing direction AND strict top is below + // last char's strict bottom (even slightly), indicating text wrapped to a new line + // within the same text object + // 3. Very large backward jump: if the char jumps back by more than the current // segment width, it's definitely a new line (handles OCR text with tall bounding // boxes that overlap vertically between lines) let strict_below = vp_strict.top > seg.last_char_bottom; - let large_leftward_jump = gap < -5.0; + let large_backward_jump = gap < -5.0; let seg_width = seg.vp_right - seg.vp_left; - let very_large_leftward_jump = seg_width > 20.0 && gap < -(seg_width * 0.5); + let very_large_backward_jump = seg_width > 20.0 && gap < -(seg_width * 0.5); let line_changed = vp_strict.top > seg.last_char_bottom + y_tolerance - || (strict_below && large_leftward_jump) - || very_large_leftward_jump; + || (strict_below && large_backward_jump) + || very_large_backward_jump; // Dot leader detection: break at the boundary between dots and non-dots. // This prevents items like "Total . . . . 330,100" from merging. @@ -1384,7 +1613,7 @@ fn extract_page_text_items( }; let split = gap >= MAX_INLINE_GAP || (seg.pending_space && gap > seg.avg_char_width() * 2.2); - let loose_gap = vp_strict.left - seg.last_char_loose_right; + let loose_gap = seg.loose_gap_to(&vp_strict, c); let em_vp = (vp_loose.bottom - vp_loose.top).abs(); let space_w = ch.font_space_width().map(|w| w * em_vp).unwrap_or(-1.0); eprintln!( @@ -1429,7 +1658,7 @@ fn extract_page_text_items( .is_some_and(|p| p.is_ascii_alphanumeric()); if prev_alnum && c.is_ascii_alphanumeric() { let em_vp = (vp_loose.bottom - vp_loose.top).abs(); - let loose_gap = vp_strict.left - seg.last_char_loose_right; + let loose_gap = seg.loose_gap_to(&vp_strict, c); if em_vp > 0.0 && loose_gap > 0.0 { let s = font_space_cal.entry(fk.clone()).or_default(); if s.len() < 512 { @@ -1454,7 +1683,7 @@ fn extract_page_text_items( // of the rendered em height as the space estimate. let em_vp = (vp_loose.bottom - vp_loose.top).abs(); let space_w = ch.font_space_width().map(|w| w * em_vp).unwrap_or(0.0); - let loose_gap = vp_strict.left - seg.last_char_loose_right; + let loose_gap = seg.loose_gap_to(&vp_strict, c); let both_alnum = c.is_ascii_alphanumeric() && seg .text @@ -1497,8 +1726,11 @@ fn extract_page_text_items( // PDFs carry the neighbouring page's text at x beyond the page edge in // the same content stream; viewers never show it. Partially-visible // items are kept. - let vb_w = (view_box.right - view_box.left).abs(); - let vb_h = (view_box.top - view_box.bottom).abs(); + // Item coordinates have already been transformed into the + // rotation-adjusted viewport. Clip against dimensions in that same space; + // using the raw CropBox dimensions here drops the right/bottom portion of + // /Rotate 90 and /Rotate 270 pages. + let (vb_w, vb_h) = page.viewport_size(view_box); let pre_clip_count = items.len(); items.retain(|it| { it.x < vb_w @@ -2102,6 +2334,16 @@ struct SegmentBuilder { last_char_right: f32, // Right edge of last char LOOSE bounds (advance-relative gap calculation) last_char_loose_right: f32, + // Left edges of the same boxes. Right-to-left runs (Hebrew, Arabic) advance + // leftward, so their trailing edge is the LEFT one; `gap_to` picks the pair + // that matches the segment's writing direction. + last_char_left: f32, + last_char_loose_left: f32, + // Writing direction of the segment, latched from its first strong-direction + // character. Stays `None` through leading neutrals (digits, punctuation) so + // a segment opening with "(" still picks up RTL from the Hebrew/Arabic + // letter that follows. + dir_rtl: Option, // Bottom of last char strict bounds (for line-change detection) last_char_bottom: f32, // Count of non-space characters (for avg width calculation) @@ -2156,6 +2398,9 @@ impl SegmentBuilder { vp_bottom: f32::MIN, last_char_right: f32::MIN, last_char_loose_right: f32::MIN, + last_char_left: f32::MAX, + last_char_loose_left: f32::MAX, + dir_rtl: None, last_char_bottom: f32::MIN, char_count: 0, unmapped_char_count: 0, @@ -2233,6 +2478,48 @@ impl SegmentBuilder { self.word_has = false; } + /// Gap from the segment's last character to the incoming one, measured + /// along the writing direction against the previous char's *trailing* edge + /// (its right edge for left-to-right text, its left edge for right-to-left). + /// Positive means the incoming char sits further along the line, negative + /// means it doubled back — the same sign convention in both directions, so + /// every threshold downstream stays direction-agnostic. + fn gap_to(&self, vp_strict: &RectF, c: char) -> f32 { + if self.dir_is_rtl(c) { + self.last_char_left - vp_strict.right + } else { + vp_strict.left - self.last_char_right + } + } + + /// As [`gap_to`](Self::gap_to), but against the previous char's LOOSE box, + /// which subtracts out intra-word kerning/overhang. This is the + /// advance-relative gap the missing-space recovery compares to the font's + /// space width. + fn loose_gap_to(&self, vp_strict: &RectF, c: char) -> f32 { + if self.dir_is_rtl(c) { + self.last_char_loose_left - vp_strict.right + } else { + vp_strict.left - self.last_char_loose_right + } + } + + /// Writing direction to use for the pair (segment tail, `c`). Falls back to + /// the incoming character while the segment has only seen neutrals. + fn dir_is_rtl(&self, c: char) -> bool { + self.dir_rtl.unwrap_or_else(|| is_rtl_char(c)) + } + + /// Record `c`'s contribution to the segment's writing direction. Neutral + /// characters (digits, punctuation, spaces) leave it untouched. + fn note_direction(&mut self, c: char) { + if is_rtl_char(c) { + self.dir_rtl = Some(true); + } else if c.is_alphabetic() { + self.dir_rtl = Some(false); + } + } + /// Average width of non-space characters in the current segment. /// Prefers actual glyph widths (text_width) over bbox width, since bbox /// includes inter-character gaps that inflate the average and cause @@ -2266,7 +2553,11 @@ impl SegmentBuilder { self.vp_bottom = vp_loose.bottom; self.last_char_right = vp_strict.right; self.last_char_loose_right = vp_loose.right; + self.last_char_left = vp_strict.left; + self.last_char_loose_left = vp_loose.left; self.last_char_bottom = vp_strict.bottom; + self.dir_rtl = None; + self.note_direction(c); self.char_count = 1; self.unmapped_char_count = if counts_as_unmapped(recovered, ch.has_unicode_map_error()) { 1 @@ -2384,7 +2675,10 @@ impl SegmentBuilder { self.vp_bottom = self.vp_bottom.max(vp_loose.bottom); self.last_char_right = vp_strict.right; self.last_char_loose_right = vp_loose.right; + self.last_char_left = vp_strict.left; + self.last_char_loose_left = vp_loose.left; self.last_char_bottom = vp_strict.bottom; + self.note_direction(c); self.char_count += 1; if self.extract_text_metadata { self.char_codes.push(ch.char_code()); @@ -2517,6 +2811,89 @@ mod tests { use super::*; use std::f32::consts::PI; + fn rotated_text_pdf() -> Vec { + let content = + b"BT /F1 10 Tf 20 40 Td (FIRSTMARK) Tj ET\nBT /F1 10 Tf 20 250 Td (SECONDMARK) Tj ET"; + let objects: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] /Rotate 90 /Resources << /Font << /F1 7 0 R >> >> /Contents 5 0 R >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] /Rotate 270 /Resources << /Font << /F1 7 0 R >> >> /Contents 6 0 R >>".to_vec(), + [ + format!("<< /Length {} >>\nstream\n", content.len()).as_bytes(), + content, + b"\nendstream", + ] + .concat(), + [ + format!("<< /Length {} >>\nstream\n", content.len()).as_bytes(), + content, + b"\nendstream", + ] + .concat(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + ]; + let mut pdf = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::with_capacity(objects.len()); + for (index, object) in objects.iter().enumerate() { + offsets.push(pdf.len()); + pdf.extend_from_slice(format!("{} 0 obj\n", index + 1).as_bytes()); + pdf.extend_from_slice(object); + pdf.extend_from_slice(b"\nendobj\n"); + } + let xref = pdf.len(); + pdf.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes()); + pdf.extend_from_slice(b"0000000000 65535 f \n"); + for offset in offsets { + pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + pdf.extend_from_slice( + format!( + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", + objects.len() + 1 + ) + .as_bytes(), + ); + pdf + } + + #[test] + fn rotated_pages_use_viewport_dimensions_and_keep_edge_text() { + let pages = + extract_pages_from_input(&PdfInput::Bytes(rotated_text_pdf()), None, usize::MAX, None) + .unwrap(); + + assert_eq!(pages.len(), 2); + for page in &pages { + assert_eq!((page.page_width, page.page_height), (300.0, 200.0)); + let raw_text = page + .text_items + .iter() + .map(|item| item.text.as_str()) + .collect::() + .replace(char::is_whitespace, ""); + assert!(raw_text.contains("FIRSTMARK"), "raw text: {raw_text}"); + assert!(raw_text.contains("SECONDMARK"), "raw text: {raw_text}"); + } + + let parsed = crate::projection::project_pages_to_grid(pages); + for page in parsed { + let text = page + .text_items + .iter() + .map(|item| item.text.as_str()) + .collect::() + .replace(char::is_whitespace, ""); + assert!(text.contains("FIRSTMARK"), "extracted text: {text}"); + assert!(text.contains("SECONDMARK"), "extracted text: {text}"); + assert!( + page.text_items + .iter() + .all(|item| item.x + item.width <= page.page_width + 0.1) + ); + } + } + // A glyph PDFium flags with a raw /ToUnicode map error normally counts // toward the item's unmapped tally... #[test] @@ -3206,4 +3583,85 @@ mod tests { ); assert!(res.is_err()); } + + #[test] + fn page_error_is_fail_fast_by_default() { + let mut page_errors = Vec::new(); + let result = resolve_page_result::<()>( + 3, + Err(LiteParseError::Other("broken page".into())), + false, + &mut page_errors, + ); + + assert!(result.is_err()); + assert!(page_errors.is_empty()); + } + + #[test] + fn page_error_can_be_collected_and_skipped() { + let mut page_errors = Vec::new(); + let result = resolve_page_result::<()>( + 3, + Err(LiteParseError::Other("broken page".into())), + true, + &mut page_errors, + ) + .unwrap(); + + assert!(result.is_none()); + assert_eq!( + page_errors, + vec![PageError { + page_number: 3, + message: "broken page".into(), + }] + ); + } + + #[test] + fn image_cache_remove_page_drops_failed_pages_renders() { + let image_ref = |id: &str, raw: &[u8]| ImageRef { + id: id.to_string(), + bbox: Rect { + x: 0.0, + y: 0.0, + width: 1.0, + height: 1.0, + }, + obj_index: 0, + format: "png".into(), + pixel_width: 1, + pixel_height: 1, + rotation: 0.0, + jpeg_bytes: None, + raw_bytes: Some(raw.to_vec()), + bits_per_pixel: 32, + colorspace: 0, + }; + let cached = |id: &str, page: u32, raw: &[u8]| CachedImage { + raw_bytes: raw.to_vec(), + id: id.to_string(), + page, + format: "png".into(), + bytes: std::sync::Arc::new(Vec::new()), + }; + + let mut cache = ImageCache::default(); + cache.insert(&image_ref("p2_1", b"two"), cached("p2_1", 2, b"two")); + cache.insert(&image_ref("p3_1", b"three"), cached("p3_1", 3, b"three")); + + // Page 2 failed after rendering: its cache entry must go so a later + // duplicate of the same bytes can't claim `duplicate_of: "p2_1"` for + // an image that was rolled back out of the output. + cache.remove_page(2); + + assert!(cache.get(&image_ref("p5_1", b"two"), b"two").is_none()); + assert_eq!( + cache + .get(&image_ref("p5_2", b"three"), b"three") + .map(|c| c.id.as_str()), + Some("p3_1") + ); + } } diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rs index 56dd89b9..4b0a4c88 100644 --- a/crates/liteparse/src/lib.rs +++ b/crates/liteparse/src/lib.rs @@ -5,14 +5,14 @@ //! // ── Public API re-exports ────────────────────────────────────────────── -pub use config::{LiteParseConfig, OutputFormat}; +pub use config::{DEFAULT_PAGE_BATCH_SIZE, LiteParseConfig, OutputFormat}; pub use error::LiteParseError; #[cfg(not(target_arch = "wasm32"))] pub use font_db_resolver::FontDbResolver; pub use glyph_resolver::{GLYPH_RESOLVER_FONT_SIZE, GlyphResolver}; -pub use parser::{LiteParse, ParseResult, ScreenshotResult}; +pub use parser::{LiteParse, ParseBatch, ParseResult, ParseSession, ScreenshotResult}; pub use search::{SearchOptions, search_items}; -pub use types::{ParsedPage, TextItem, WordBox}; +pub use types::{DocumentMetadata, ParsedPage, TextItem, WordBox}; // ── Modules with user-facing types (visible in docs) ─────────────────── pub mod config; @@ -25,9 +25,11 @@ pub mod types; // ── Internal modules (available for binding crates, hidden from docs) ── #[cfg(not(target_arch = "wasm32"))] mod acroform_repair; +mod bidi; #[cfg(not(target_arch = "wasm32"))] #[doc(hidden)] pub mod conversion; +mod document_metadata; #[doc(hidden)] pub mod extract; #[doc(hidden)] @@ -49,6 +51,5 @@ pub mod ocr_merge; pub mod output; #[doc(hidden)] pub mod projection; -#[cfg(not(target_arch = "wasm32"))] #[doc(hidden)] pub mod render; diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rs index 48cb1912..df6c4269 100644 --- a/crates/liteparse/src/main.rs +++ b/crates/liteparse/src/main.rs @@ -79,6 +79,10 @@ struct ParseCommand { #[arg(long)] target_pages: Option, + /// Continue after page-level extraction errors and report them in JSON. + #[arg(long)] + continue_on_page_error: bool, + /// DPI for rendering (default: 150) #[arg(long, default_value = "150")] dpi: f32, @@ -226,6 +230,10 @@ struct BatchParseCommand { #[arg(long, default_value = "1000")] max_pages: usize, + /// Continue after page-level extraction errors and report them in JSON. + #[arg(long)] + continue_on_page_error: bool, + /// DPI for rendering #[arg(long, default_value = "150")] dpi: f32, @@ -373,6 +381,24 @@ fn parse_image_mode(s: &str) -> Result { } } +/// Surface tolerated page failures on stderr. JSON output carries +/// `page_errors` itself, but text/markdown would otherwise silently omit the +/// failed pages, so this prints unconditionally (not gated on `--quiet`). +fn warn_page_errors(result: &liteparse::parser::ParseResult, file: Option<&str>) { + for error in &result.page_errors { + match file { + Some(file) => eprintln!( + "[liteparse] {}: page {} failed to extract and was skipped: {}", + file, error.page_number, error.message + ), + None => eprintln!( + "[liteparse] page {} failed to extract and was skipped: {}", + error.page_number, error.message + ), + } + } +} + #[tokio::main] async fn main() -> Result<(), Box> { let cli = Cli::parse(); @@ -388,6 +414,7 @@ async fn main() -> Result<(), Box> { tessdata_path: cmd.tessdata_path, max_pages: cmd.max_pages, target_pages: cmd.target_pages, + continue_on_page_error: cmd.continue_on_page_error, dpi: cmd.dpi, output_format: format, preserve_very_small_text: cmd.preserve_small_text, @@ -420,6 +447,7 @@ async fn main() -> Result<(), Box> { } else { lp.parse(&cmd.file).await? }; + warn_page_errors(&result, None); let formatted = match lp.config().output_format { OutputFormat::Json => { json::format_json_result(&result, lp.config().extract_text_metadata)? @@ -491,6 +519,7 @@ async fn main() -> Result<(), Box> { tessdata_path: cmd.tessdata_path, max_pages: cmd.max_pages, target_pages: None, + continue_on_page_error: cmd.continue_on_page_error, dpi: cmd.dpi, output_format: format.clone(), preserve_very_small_text: false, @@ -548,6 +577,7 @@ async fn main() -> Result<(), Box> { match lp.parse(file_path).await { Ok(result) => { + warn_page_errors(&result, Some(file_path)); let fmt_result: Result> = match lp.config().output_format { OutputFormat::Json => json::format_json_result( diff --git a/crates/liteparse/src/markdown_layout/classify.rs b/crates/liteparse/src/markdown_layout/classify.rs index 41faa58f..bb6b45cf 100644 --- a/crates/liteparse/src/markdown_layout/classify.rs +++ b/crates/liteparse/src/markdown_layout/classify.rs @@ -19,7 +19,7 @@ use super::paragraphs::{ continues_paragraph, ends_hyphenated, ends_sentence_final, is_soft_hyphen_break, }; use super::repetition::is_header_or_footer; -use super::tables::{detect_ruled_tables, detect_tables, merge_table_runs}; +use super::tables::{detect_ruled_tables, detect_tables_banded, merge_table_runs}; /// A document-order page interruption that breaks the normal text flow: either /// a horizontal rule (from vector graphics) or a figure injection (a raster @@ -160,12 +160,15 @@ pub fn classify_page_with_filters( // the region pipeline, and emit each table as a y-positioned interruption. // Runs before cross-region merge so that pass (and the region indices its // runs carry) operate on the already-filtered line list. + // Rule segments are extracted from the page graphics once and shared by + // every table-detection pass below (global, per-region, leaf veto, bands). + let rule_segments = super::tables::extract_rule_segments(&page.graphics); let mut global_ruled_tables: Vec<(f32, Block)> = Vec::new(); let mut global_ruled_consumed: std::collections::HashSet = std::collections::HashSet::new(); for (run, consumed) in super::tables::detect_ruled_tables_global( lines, - &page.graphics, + &rule_segments, page.page_width, page.page_height, ) { @@ -186,14 +189,20 @@ pub fn classify_page_with_filters( // override (a decorative frame can span a data table plus surrounding // prose; the data region tables cleanly by itself, while the frame // would fuse prose into garbage cells). + // The leaf must also hold a material share of the run. A leaf with a + // handful of the lines - a landscape slide's title band sitting above + // the table, which tables "successfully" on its own as a row of + // gutter-split title fragments - is not evidence that the per-region + // path has this content covered; vetoing on it drops the whole table + // and spills its body rows into prose. let already_handled = groups.values().any(|idxs| { - if idxs.len() < 2 { + if idxs.len() < 2 || idxs.len() * 4 < consumed.len() { return false; } let sub: Vec = idxs.iter().map(|&i| lines[i].clone()).collect(); !super::tables::detect_ruled_tables( &sub, - &page.graphics, + &rule_segments, page.page_width, page.page_height, ) @@ -374,6 +383,7 @@ pub fn classify_page_with_filters( region_lines, region_interruptions, page, + &rule_segments, heading_map, outline, toc_page, @@ -477,6 +487,7 @@ fn classify_region( lines: &[ProjectedLine], interruptions: Vec<(f32, Interruption)>, page: &ParsedPage, + rule_segments: &super::tables::RuleSegments, heading_map: &[(f32, u8)], outline: &[OutlineTarget], toc_page: bool, @@ -496,8 +507,9 @@ fn classify_region( // graphics are still consulted for ruled-table detection because path // objects are page-coordinate; the detector intersects them against the // sub-list's line bboxes anyway. - let ruled_runs = detect_ruled_tables(lines, &page.graphics, page.page_width, page.page_height); - let borderless_runs = precomputed_tables.unwrap_or_else(|| detect_tables(lines)); + let ruled_runs = detect_ruled_tables(lines, rule_segments, page.page_width, page.page_height); + let borderless_runs = precomputed_tables + .unwrap_or_else(|| detect_tables_banded(lines, rule_segments, page.page_height)); let table_runs = merge_table_runs(ruled_runs, borderless_runs); // Region-wide pre-pass: which line indices carry a lettered/roman marker diff --git a/crates/liteparse/src/markdown_layout/cross_region.rs b/crates/liteparse/src/markdown_layout/cross_region.rs index a0472ce2..77246c66 100644 --- a/crates/liteparse/src/markdown_layout/cross_region.rs +++ b/crates/liteparse/src/markdown_layout/cross_region.rs @@ -228,14 +228,28 @@ fn cluster_rows<'a>(set: &[(&'a ProjectedLine, bool)], tol: f32) -> Vec ProjectedLine { let mut members: Vec<&ProjectedLine> = cluster.members().copied().collect(); - members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x)); + // Fuse in reading order: a right-to-left row starts at its highest x, so + // walking left-to-right would emit its cells backwards. `fused.spans` is + // re-sorted x-ascending at the end either way, keeping cell geometry intact. + let rtl = crate::bidi::is_rtl_pieces(members.iter().map(|m| m.text.as_str())); + if rtl { + members.sort_by(|a, b| b.bbox.x.total_cmp(&a.bbox.x)); + } else { + members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x)); + } let first = members[0]; let mut fused = first.clone(); + fused.rtl = rtl; fused.region_path = path.to_vec(); for m in &members[1..] { - let prev_right = fused.bbox.x + fused.bbox.width; - let gap = (m.bbox.x - prev_right).max(0.0); + // Gap to the previously-fused run, measured along the reading + // direction so the space count stays proportional in both. + let gap = if rtl { + (fused.bbox.x - (m.bbox.x + m.bbox.width)).max(0.0) + } else { + (m.bbox.x - (fused.bbox.x + fused.bbox.width)).max(0.0) + }; let approx_char_w = (fused.dominant_font_size * 0.5).max(2.0); let n_spaces = ((gap / approx_char_w) as usize).clamp(2, 24); fused.text.push_str(&" ".repeat(n_spaces)); diff --git a/crates/liteparse/src/markdown_layout/flags.rs b/crates/liteparse/src/markdown_layout/flags.rs index 3fdab7e0..9ca25c5c 100644 --- a/crates/liteparse/src/markdown_layout/flags.rs +++ b/crates/liteparse/src/markdown_layout/flags.rs @@ -18,3 +18,4 @@ env_set_flag!(DEBUG_MD, "LITEPARSE_DEBUG_MD"); env_set_flag!(DEBUG_TABLE, "LITEPARSE_DEBUG_TABLE"); env_set_flag!(DEBUG_RULED, "LITEPARSE_DEBUG_RULED"); env_set_flag!(DEBUG_CROSS_REGION, "LITEPARSE_DEBUG_CROSS_REGION"); +env_set_flag!(DEBUG_GUTTER, "LITEPARSE_DEBUG_GUTTER"); diff --git a/crates/liteparse/src/markdown_layout/inline.rs b/crates/liteparse/src/markdown_layout/inline.rs index e0c1e0d7..f43edbd6 100644 --- a/crates/liteparse/src/markdown_layout/inline.rs +++ b/crates/liteparse/src/markdown_layout/inline.rs @@ -44,6 +44,26 @@ pub(super) fn escape_inline(s: &str) -> String { out } +/// Escape `text` for use as the body of a span with `style`. Backslash escapes +/// are inert inside code spans per CommonMark, so mono text is passed through +/// verbatim. +fn style_body(text: &str, style: SpanStyle) -> String { + if style.mono { + text.to_string() + } else { + escape_inline(text) + } +} + +/// Render `text` as a single styled markdown span: style-aware escaping +/// (`style_body`) plus marker wrapping (`apply_style`). Always use this pair +/// through here — calling `escape_inline` + `apply_style` directly would +/// backslash-escape the body of a code span, which CommonMark renders +/// literally. +fn render_span(text: &str, style: SpanStyle) -> String { + apply_style(&style_body(text, style), style) +} + /// Wrap `inner` in a markdown inline link to `url`. Uses the angle-bracket /// destination form when the URL contains characters that would otherwise /// terminate or break the `(url)` form (whitespace or parentheses). @@ -60,12 +80,24 @@ fn apply_link(inner: &str, url: &str) -> String { /// a span is mono we drop the `**/*` wrap. Bold + italic → `***…***`. fn apply_style(inner: &str, style: SpanStyle) -> String { let styled = if style.mono { - // Use backticks; if inner already contains backticks, switch to a - // longer fence (pair of backticks plus a space buffer) per CommonMark. - if inner.contains('`') { - format!("`` {} ``", inner) - } else { + // The fence must be longer than the longest backtick run inside the + // content (a matching-length interior run would close the span early), + // with a space buffer so an edge backtick can't merge into the fence. + // CommonMark strips one space from each end, so the content round-trips. + let longest_run = { + let mut longest = 0usize; + let mut run = 0usize; + for c in inner.chars() { + run = if c == '`' { run + 1 } else { 0 }; + longest = longest.max(run); + } + longest + }; + if longest_run == 0 { format!("`{}`", inner) + } else { + let fence = "`".repeat(longest_run + 1); + format!("{fence} {inner} {fence}") } } else { match (style.bold, style.italic) { @@ -105,10 +137,17 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String { return collapse_whitespace(&line.text); } - // Sort spans by x so we render in visual reading order regardless of - // extraction order. Stable so equal-x spans keep their original sequence. + // Sort spans into reading order regardless of extraction order. Stable so + // equal-x spans keep their original sequence. A right-to-left line reads + // from the highest x down, matching the join `build_one_line` used for + // `line.text` — the uniform-style shortcut below falls back on that string, + // so the two orderings have to agree. let mut spans = spans; - spans.sort_by(|a, b| a.x.total_cmp(&b.x)); + if line.rtl { + spans.sort_by(|a, b| b.x.total_cmp(&a.x)); + } else { + spans.sort_by(|a, b| a.x.total_cmp(&b.x)); + } let styles: Vec = spans.iter().map(|s| SpanStyle::from_item(s)).collect(); let links: Vec> = spans.iter().map(|s| s.link.as_deref()).collect(); @@ -122,11 +161,7 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String { if joined.is_empty() { return joined; } - let escaped = escape_inline(&joined); - if styles[0].is_plain() { - return escaped; - } - return apply_style(&escaped, styles[0]); + return render_span(&joined, styles[0]); } // Group consecutive spans by style. Within a group, span texts join with @@ -148,12 +183,7 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String { group_text.push_str(span.text.trim()); } let group_text = collapse_whitespace(&group_text); - let escaped = escape_inline(&group_text); - let mut rendered = if style.is_plain() { - escaped - } else { - apply_style(&escaped, style) - }; + let mut rendered = render_span(&group_text, style); if let Some(url) = link { rendered = apply_link(&rendered, url); } @@ -179,13 +209,7 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String { /// around it). On any failure we fall back to plain escaped `rest`. pub(super) fn render_list_item_text(line: &ProjectedLine, marker: &str, rest: &str) -> String { if let Some(style) = line_uniform_style(line) { - let plain = collapse_whitespace(rest); - let escaped = escape_inline(&plain); - return if style.is_plain() { - escaped - } else { - apply_style(&escaped, style) - }; + return render_span(&collapse_whitespace(rest), style); } let full = render_line_inline(line); if let Some(stripped) = strip_leading_marker_from_inline(&full, marker) { @@ -415,4 +439,58 @@ mod tests { assert!(out.contains("call")); assert!(out.contains("on it")); } + + #[test] + fn render_line_inline_mono_span_is_not_escaped() { + let l = styled_line( + &[ + ("call", 50.0, Some("Arial")), + ("get_user_id", 100.0, Some("Courier")), + ("now", 200.0, Some("Arial")), + ], + 100.0, + 10.0, + ); + let out = render_line_inline(&l); + assert!(out.contains("`get_user_id`"), "got: {out}"); + assert!(!out.contains('\\'), "got: {out}"); + } + + #[test] + fn render_line_inline_mono_span_keeps_backslashes() { + let l = styled_line( + &[ + ("open", 50.0, Some("Arial")), + (r"C:\Program\bin", 100.0, Some("Courier")), + ("next", 200.0, Some("Arial")), + ], + 100.0, + 10.0, + ); + let out = render_line_inline(&l); + assert!(out.contains(r"`C:\Program\bin`"), "got: {out}"); + } + + #[test] + fn render_line_inline_uniform_mono_line_is_not_escaped() { + let l = styled_line(&[("a_b*c", 50.0, Some("Courier"))], 100.0, 10.0); + let out = render_line_inline(&l); + assert_eq!(out, "`a_b*c`"); + } + + #[test] + fn mono_span_with_single_backtick_uses_double_fence() { + let l = styled_line(&[("a`b", 50.0, Some("Courier"))], 100.0, 10.0); + let out = render_line_inline(&l); + assert_eq!(out, "`` a`b ``"); + } + + #[test] + fn mono_span_with_double_backtick_run_uses_longer_fence() { + // A 2-backtick run inside the content would close a 2-backtick fence + // early; the fence must be one longer than the longest interior run. + let l = styled_line(&[("a``b", 50.0, Some("Courier"))], 100.0, 10.0); + let out = render_line_inline(&l); + assert_eq!(out, "``` a``b ```"); + } } diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs index b0ca3b27..7fa0ff8a 100644 --- a/crates/liteparse/src/markdown_layout/tables.rs +++ b/crates/liteparse/src/markdown_layout/tables.rs @@ -31,6 +31,14 @@ const TABLE_MIN_TRACK_GAP_FLOOR_PT: f32 = 12.0; /// anchor the header/body split in ruled-grid collapse. const TABLE_ROW_MIN_FILL: f32 = 0.9; +/// Two horizontal rules closer than this bound a heading underline or a boxed +/// caption, not a table body. +const RULE_BAND_MIN_HEIGHT_PT: f32 = 10.0; + +/// A rule band with fewer lines than this is a rule under a heading or a run of +/// hyperlink underlines - never a table. +const RULE_BAND_MIN_LINES: usize = 3; + /// Floor for the sparse-new-row path: a partial-cell line whose bottom-gap /// exceeds this fraction qualifies as a real new row (with empty cells at /// missing tracks) instead of being treated as a wrap continuation. Below @@ -46,6 +54,198 @@ const TABLE_ROW_GAP_MULTIPLIER: f32 = 2.5; /// (rejecting irregular spacing that's more likely prose or a footer block). const TABLE_ROW_SPACING_MAX_CV: f32 = 0.5; +/// Minimum ratio between the smallest "wide" inter-word gap and the largest +/// ordinary one for a span's internal gaps to read as bimodal - i.e. for the +/// wide ones to be column gutters rather than stretched justification spaces. +/// +/// Measured separation is large: real gutters run 3.4–4.4× the in-cell word +/// gap (7.29 vs 2.13 on a booktabs worksheet header, 7.73 vs 1.76 on a +/// two-column-label results table), while fully-justified prose - the one +/// thing that reliably counterfeits column structure - tops out around 1.3×. +const SPAN_GUTTER_MIN_RATIO: f32 = 2.5; + +/// Absolute floor (points) on a gap treated as an in-span column gutter, so +/// tightly-kerned small text can't manufacture columns out of a 1pt jitter. +const SPAN_GUTTER_MIN_PT: f32 = 3.0; + +/// Floor on the *ordinary* side of the bimodality ratio, as a fraction of the +/// run's font size. The ratio is meant to compare candidate gutters against +/// typical word spacing, but the sorted-gap scan takes whatever gap sits below +/// the jump - and one degenerate sub-point gap (kerning jitter, a run-on +/// superscript) would make an ordinary 3pt word space clear the ratio and read +/// as a gutter tier of its own. A word space in `f`pt type is never far below +/// `0.2 × f`, so nothing smaller may serve as the comparison base. +const SPAN_GUTTER_MIN_ORDINARY_EM: f32 = 0.2; + +/// One horizontal piece of a PDFium text run: the run split at its internal +/// column gutters. Most runs yield a single piece. +#[derive(Debug, Clone)] +pub(super) struct SpanPiece<'a> { + pub(super) x: f32, + pub(super) end_x: f32, + pub(super) text: String, + /// The run this piece came from - for bold/font lookups. + pub(super) span: &'a TextItem, + /// The piece's own word boxes, in reading order, but **only** when they + /// were verified to reconstruct `text` exactly. Empty otherwise, so a + /// consumer can treat non-empty as "real geometry for every word here". + pub(super) words: Vec<&'a crate::types::WordBox>, +} + +/// A run's own word boxes, in reading order, but only when they reconstruct the +/// run's text exactly. Anything less and the boxes can't be used to slice the +/// text, because the split points wouldn't correspond to it. +/// +/// The x-filter matters: projection can append a merged neighbour's word boxes +/// onto a run without re-slicing the text, making `span.words` a superset. +pub(super) fn verified_words(span: &TextItem) -> Option> { + let x1 = span.x + span.width.max(0.0); + let words: Vec<&crate::types::WordBox> = span + .words + .iter() + .filter(|w| !w.text.trim().is_empty() && w.x >= span.x - 1.0 && w.x <= x1 + 1.0) + .collect(); + if words.is_empty() { + return None; + } + let rebuilt = collapse_whitespace( + &words + .iter() + .map(|w| w.text.trim()) + .collect::>() + .join(" "), + ); + (rebuilt == collapse_whitespace(span.text.trim())).then_some(words) +} + +/// Split one PDFium run into column pieces at its internal gutters. +/// +/// PDFium routinely emits a whole table row (several cells) as a single run, +/// which is the root of the track chicken-and-egg: the detector needs tracks to +/// split runs, but the runs are where the track evidence lives. Word boxes +/// break the cycle, because the gutter is plainly visible in the geometry +/// before any table hypothesis exists. +/// +/// Detection is *relative to the run itself*, never a constant threshold: sort +/// the inter-word gaps, find the largest ratio jump, and accept the split only +/// when that jump clears [`SPAN_GUTTER_MIN_RATIO`]. A fixed gap threshold +/// cannot work here: the same 4pt gap is a gutter in 6pt type and an ordinary +/// space in 12pt type. +/// +/// Returns a single piece covering the whole run when it has no word boxes, +/// fewer than three words, or no bimodal gap. +pub(super) fn split_span_at_gutters(span: &TextItem) -> Vec> { + // Unverified word boxes leave `words` empty, so `whole()` then carries no + // word geometry, exactly as the `SpanPiece::words` contract requires. + let words: Vec<&crate::types::WordBox> = verified_words(span).unwrap_or_default(); + let whole = || { + vec![SpanPiece { + x: span.x, + end_x: span.x + span.width.max(0.0), + text: collapse_whitespace(span.text.trim()), + span, + words: words.clone(), + }] + }; + // Two words give exactly one gap, which is trivially "the largest": no + // ratio to test against, so there is no evidence of bimodality. The piece + // still carries its words, so a track-driven split can use them later. + if words.len() < 3 { + return whole(); + } + + let gaps: Vec = words + .windows(2) + .map(|w| w[1].x - (w[0].x + w[0].width.max(0.0))) + .collect(); + let mut sorted = gaps.clone(); + sorted.sort_by(f32::total_cmp); + // Largest ratio jump between adjacent sorted gaps: the boundary between + // "ordinary space" and "gutter", if there is one. + let font = span.font_size.unwrap_or(span.height).max(1.0); + let lo_floor = font * SPAN_GUTTER_MIN_ORDINARY_EM; + let mut cut = None; + let mut best_ratio = SPAN_GUTTER_MIN_RATIO; + for i in 0..sorted.len() - 1 { + let (lo, hi) = (sorted[i], sorted[i + 1]); + if hi < SPAN_GUTTER_MIN_PT { + continue; + } + let ratio = hi / lo.max(lo_floor); + if ratio >= best_ratio { + best_ratio = ratio; + cut = Some(hi); + } + } + let Some(threshold) = cut else { + return whole(); + }; + + let mut pieces: Vec> = Vec::new(); + let mut current: Vec<&crate::types::WordBox> = vec![words[0]]; + for (i, gap) in gaps.iter().enumerate() { + if *gap >= threshold { + pieces.push(piece_from_words(¤t, span)); + current.clear(); + } + current.push(words[i + 1]); + } + pieces.push(piece_from_words(¤t, span)); + pieces.retain(|p| !p.text.is_empty()); + if pieces.len() < 2 { + return whole(); + } + if *super::flags::DEBUG_GUTTER { + eprintln!( + "[gutter] thr={:.2} ratio={:.2} {:?}", + threshold, + best_ratio, + pieces.iter().map(|p| p.text.as_str()).collect::>() + ); + } + pieces +} + +fn piece_from_words<'a>(words: &[&'a crate::types::WordBox], span: &'a TextItem) -> SpanPiece<'a> { + let x = words.first().map_or(span.x, |w| w.x); + let end_x = words + .last() + .map_or(span.x + span.width.max(0.0), |w| w.x + w.width.max(0.0)); + SpanPiece { + x, + end_x, + text: collapse_whitespace( + &words + .iter() + .map(|w| w.text.trim()) + .collect::>() + .join(" "), + ), + span, + words: words.to_vec(), + } +} + +/// All column pieces on a line, left to right. This is the tabular view of a +/// row: what PDFium calls a run is not what the page calls a cell. +pub(super) fn line_pieces(line: &ProjectedLine) -> Vec> { + let mut pieces: Vec> = line + .spans + .iter() + .filter(|s| !s.text.trim().is_empty()) + .flat_map(split_span_at_gutters) + .collect(); + pieces.sort_by(|a, b| a.x.total_cmp(&b.x)); + pieces +} + +/// `line_pieces` for every line up front. The detection scan tries most lines +/// as a seed and re-reads its neighbours' pieces for each try, so computing +/// them per call would redo the same splits O(lines × window) times. +fn compute_line_pieces(lines: &[ProjectedLine]) -> Vec>> { + lines.iter().map(line_pieces).collect() +} + /// One cell within a tabular row: contributing spans aggregated to text and /// its leftmost x position, used to align cells across rows into column /// "tracks". @@ -341,29 +541,29 @@ const TABLE_TRACK_INFERENCE_MAX_ROWS: usize = 12; /// 13.9pt inter-item gaps). It also surfaces tracks witnessed by even a /// single row when other rows in the same table have PDFium-level merged /// spans that hide the full column geometry. -fn infer_tracks_from_raw_items(lines: &[ProjectedLine], start_idx: usize) -> Vec { +fn infer_tracks_from_raw_items( + lines: &[ProjectedLine], + pieces: &[Vec>], + start_idx: usize, +) -> Vec { let mut xs: Vec = Vec::new(); - let push_row_xs = |xs: &mut Vec, line: &ProjectedLine| { - let row_xs: Vec = line - .spans - .iter() - .filter(|s| !s.text.trim().is_empty()) - .map(|s| s.x) - .collect(); + let push_row_xs = |xs: &mut Vec, idx: usize| { + // Piece x's, not span x's: a row PDFium emitted as one merged run + // still witnesses its columns through its internal gutters. // Skip 0- or 1-item rows — they don't carry column info and can // introduce noise from single-cell prose lines. - if row_xs.len() >= 2 { - xs.extend(row_xs); + if pieces[idx].len() >= 2 { + xs.extend(pieces[idx].iter().map(|p| p.x)); } }; - push_row_xs(&mut xs, &lines[start_idx]); + push_row_xs(&mut xs, start_idx); let mut j = start_idx + 1; let mut rows_used = 1; while j < lines.len() && rows_used < TABLE_TRACK_INFERENCE_MAX_ROWS { if !table_rows_adjacent(&lines[j - 1], &lines[j]) { break; } - push_row_xs(&mut xs, &lines[j]); + push_row_xs(&mut xs, j); j += 1; rows_used += 1; } @@ -419,18 +619,21 @@ fn cells_from_raw_items_with_tracks( line: &ProjectedLine, tracks: &[f32], ) -> Option> { - let mut spans: Vec<&TextItem> = line - .spans - .iter() - .filter(|s| !s.text.trim().is_empty()) - .collect(); - spans.sort_by(|a, b| a.x.total_cmp(&b.x)); - // Require ≥ 2 PDFium spans on the row. A 1-span row spanning multiple - // tracks is almost always prose (wrapped paragraph whose x-range - // happens to overlap the track region); shredding it at whitespace - // anchors corrupts the body text. Real merged-numeric table rows still - // have a label span and a values span (≥ 2). - if spans.len() < 2 { + cells_from_pieces(&line_pieces(line), tracks, false) +} + +/// `allow_single_piece` admits a row holding one piece. Off by default: a +/// single piece spanning multiple tracks is almost always prose (a wrapped +/// paragraph whose x-range merely overlaps the track region), and shredding it +/// at whitespace anchors corrupts the body text. It is only safe where the row +/// is known to be inside a table already - a rule band whose second column is +/// blank, where every body row legitimately has one piece. +fn cells_from_pieces( + spans: &[SpanPiece<'_>], + tracks: &[f32], + allow_single_piece: bool, +) -> Option> { + if spans.len() < 2 && !(allow_single_piece && spans.len() == 1) { return None; } let tol = TABLE_TRACK_TOLERANCE_PT; @@ -453,9 +656,9 @@ fn cells_from_raw_items_with_tracks( } dst.push_str(src); }; - for span in &spans { + for span in spans { let x0 = span.x; - let x1 = span.x + span.width.max(0.0); + let x1 = span.end_x; let covered: Vec = tracks .iter() .enumerate() @@ -480,13 +683,18 @@ fn cells_from_raw_items_with_tracks( let idx = covered[0]; push_text(&mut cells[idx].text, &span.text); cells[idx].end_x = cells[idx].end_x.max(x1); - if is_bold_item(span) { + if is_bold_item(span.span) { cells[idx].bold = true; } } _ => { - let pieces = split_span_at_anchors(span, &covered, tracks)?; - let bold = is_bold_item(span); + let anchors: Vec = covered[1..].iter().map(|&i| tracks[i]).collect(); + // Real word x's first; the character-index estimate only when + // this piece has no verified word geometry to cut on. + let pieces = split_words_at_x_anchors(&span.words, &anchors).or_else(|| { + split_text_at_x_anchors(&span.text, span.x, span.end_x - span.x, &anchors) + })?; + let bold = is_bold_item(span.span); for (idx, piece) in covered.iter().zip(pieces.iter()) { if piece.is_empty() { return None; @@ -541,6 +749,55 @@ fn is_value_like(text: &str) -> bool { /// usable whitespace boundary (e.g. unbroken text like a long hex string). /// Pieces may be empty strings — callers that require non-empty pieces must /// check. +/// Split a word sequence into `anchors.len() + 1` pieces using the words' own +/// x positions: each anchor picks the inter-word boundary whose following word +/// starts closest to it. +/// +/// This is the geometric counterpart of [`split_text_at_x_anchors`], which can +/// only *interpolate* an x from a character index and so assumes every glyph is +/// the same width - false for any proportional font, and the reason a split can +/// land one character off (`10 .1%`). Prefer this whenever the piece carries +/// verified word boxes. +/// +/// Returns `None` when there are too few words to host every anchor, when two +/// anchors want the same boundary, or when an anchor's nearest boundary is +/// further away than the track tolerance - a miss that large means no word +/// actually starts at this column, so the caller should fall back rather than +/// cut where the geometry says nothing happens. +fn split_words_at_x_anchors( + words: &[&crate::types::WordBox], + anchors: &[f32], +) -> Option> { + if anchors.is_empty() || words.len() < anchors.len() + 1 { + return None; + } + let mut cuts: Vec = Vec::with_capacity(anchors.len()); + for &target in anchors { + let (k, dist) = (1..words.len()) + .filter(|k| !cuts.contains(k)) + .map(|k| (k, (words[k].x - target).abs())) + .min_by(|a, b| a.1.total_cmp(&b.1))?; + if dist > TABLE_TRACK_TOLERANCE_PT { + return None; + } + cuts.push(k); + } + cuts.sort_unstable(); + let mut pieces: Vec = Vec::with_capacity(cuts.len() + 1); + let mut prev = 0usize; + for &k in cuts.iter().chain(std::iter::once(&words.len())) { + pieces.push(collapse_whitespace( + &words[prev..k] + .iter() + .map(|w| w.text.trim()) + .collect::>() + .join(" "), + )); + prev = k; + } + Some(pieces) +} + fn split_text_at_x_anchors( text: &str, x0: f32, @@ -650,6 +907,12 @@ fn finalize_table_run( if header.is_none() && body_rows.len() < TABLE_MIN_ROWS { return None; } + // A header with no body is not a table. Reachable only via the bold-first-row + // promotion, which consumes rows[0] - harmless before the two-row relaxation, + // but with it rows[0] can be the *only* row. + if body_rows.is_empty() { + return None; + } if *super::flags::DEBUG_TABLE { eprintln!( @@ -668,11 +931,20 @@ fn finalize_table_run( }) } +/// `allow_two_row` relaxes the two body-length gates below so a header row plus +/// a *single* data row can form a table. Only ever set by the last-resort second +/// pass in `detect_tables_impl`, which runs exclusively in regions where the +/// normal pass found no table at all - see the comment there for why relaxing +/// these gates globally is unsafe. fn try_detect_table_inferred( lines: &[ProjectedLine], + pieces: &[Vec>], start_idx: usize, floor: usize, + allow_two_row: bool, + allow_two_col: bool, ) -> Option { + let min_columns = if allow_two_col { 2 } else { TABLE_MIN_COLUMNS }; let dbgt = *super::flags::DEBUG_TABLE; let seed_txt: String = lines[start_idx] .spans @@ -690,7 +962,7 @@ fn try_detect_table_inferred( } let baseline_cells = split_cells(&lines[start_idx]); - let tracks = infer_tracks_from_raw_items(lines, start_idx); + let tracks = infer_tracks_from_raw_items(lines, pieces, start_idx); if dbgt { eprintln!( "[tbl-inferred try @{start_idx} \"{:.40}\"] tracks={} baseline={} xs=[{}]", @@ -704,7 +976,7 @@ fn try_detect_table_inferred( .join(",") ); } - if tracks.len() < TABLE_MIN_COLUMNS { + if tracks.len() < min_columns { bail!("tracks {} < MIN_COLUMNS", tracks.len()); } // Only bother if we'd actually unlock more columns than the default path. @@ -755,18 +1027,14 @@ fn try_detect_table_inferred( // on the header, the run would break at the first unalignable header cell, // and the table would fall to the header-seeded path that drops a column. let tol = TABLE_TRACK_TOLERANCE_PT; - let is_strong_row = |line: &ProjectedLine| -> bool { - let spans: Vec<&TextItem> = line - .spans - .iter() - .filter(|s| !s.text.trim().is_empty()) - .collect(); + let is_strong_row = |idx: usize| -> bool { + let spans = &pieces[idx]; if spans.len() < tracks.len() { return false; } spans.iter().all(|s| { let x0 = s.x; - let x1 = s.x + s.width.max(0.0); + let x1 = s.end_x; tracks .iter() .filter(|&&t| t >= x0 - tol && t <= x1 + tol) @@ -782,7 +1050,7 @@ fn try_detect_table_inferred( if k > start_idx && !table_rows_adjacent(&lines[k - 1], &lines[k]) { break; } - if is_strong_row(&lines[k]) { + if is_strong_row(k) { body_start = Some(k); break; } @@ -793,10 +1061,10 @@ fn try_detect_table_inferred( let Some(body_start) = body_start else { bail!("no strong body row in window"); }; - let Some(first) = cells_from_raw_items_with_tracks(&lines[body_start], &tracks) else { + let Some(first) = cells_from_pieces(&pieces[body_start], &tracks, allow_two_col) else { bail!("body row cells unassignable"); }; - if first.iter().filter(|c| !c.text.is_empty()).count() < TABLE_MIN_COLUMNS { + if first.iter().filter(|c| !c.text.is_empty()).count() < min_columns { bail!("body populated cells < MIN_COLUMNS"); } let mut rows: Vec<(usize, &ProjectedLine, Vec)> = @@ -811,7 +1079,7 @@ fn try_detect_table_inferred( if !table_rows_adjacent(rows.last().unwrap().1, &lines[j]) { break; } - let Some(cells) = cells_from_raw_items_with_tracks(&lines[j], &tracks) else { + let Some(cells) = cells_from_pieces(&pieces[j], &tracks, allow_two_col) else { if dbgt { let rt: String = lines[j] .spans @@ -831,7 +1099,8 @@ fn try_detect_table_inferred( rows.push((j, &lines[j], cells)); j += 1; } - if rows.len() < TABLE_MIN_ROWS { + let min_rows = if allow_two_row { 1 } else { TABLE_MIN_ROWS }; + if rows.len() < min_rows { bail!("rows {} < MIN_ROWS", rows.len()); } // When the body was seeded below `start_idx` (the lead line was a header @@ -841,7 +1110,7 @@ fn try_detect_table_inferred( // header-seeded path of the real table below it. Already-strong seeds // (body_start == start_idx) keep the // standard MIN_ROWS threshold and are unaffected. - if body_start > start_idx && rows.len() < 3 { + if body_start > start_idx && rows.len() < 3 && !allow_two_row { bail!("advanced body_start but only {} rows", rows.len()); } let cv = row_spacing_cv(&rows); @@ -852,7 +1121,7 @@ fn try_detect_table_inferred( let end = j; let bold_eligible = rows[0].2.iter().all(|c| c.bold && !c.text.is_empty()); - finalize_table_run( + let run = finalize_table_run( lines, body_start, floor, @@ -861,7 +1130,63 @@ fn try_detect_table_inferred( column_count, end, bold_eligible, - ) + )?; + // Runs that exist *only* because the relaxation fired must clear the extra + // content gates. Runs that would have passed the normal floor anyway are + // untouched, so this can never reject something the normal pass accepted. + if allow_two_row && rows.len() < TABLE_MIN_ROWS && !two_row_run_plausible(lines, &run) { + bail!("two-row run failed plausibility gates"); + } + Some(run) +} + +/// Plausibility gates for a header + single-data-row run. +/// +/// Two rows is the weakest possible structural signal, and there is one thing +/// that reliably counterfeits it: **fully-justified prose**. Justification +/// stretches inter-word spaces until they read as column gutters, so any two +/// consecutive lines of a justified paragraph infer clean tracks and shred into +/// `| when travelling | to | conflict | zones, | more |`. +/// +/// Two signals separate the real thing from the counterfeit: +/// +/// 1. **Isolation.** A real 2-row table is surrounded by whitespace. A prose +/// pair is mid-paragraph, so its neighbours are table-adjacent lines. +/// 2. **Header shape.** Header cells are labels: they start with a capital or +/// a digit and don't trail mid-sentence punctuation. Prose "headers" are +/// lowercase words, often ending in a comma. +fn two_row_run_plausible(lines: &[ProjectedLine], run: &TableRun) -> bool { + let Block::Table { + header: Some(header), + .. + } = &run.block + else { + return false; + }; + + if run.start > 0 && table_rows_adjacent(&lines[run.start - 1], &lines[run.start]) { + return false; + } + if run.end < lines.len() && table_rows_adjacent(&lines[run.end - 1], &lines[run.end]) { + return false; + } + + let mut non_empty = 0; + for cell in header { + let t = cell.trim(); + if t.is_empty() { + continue; + } + non_empty += 1; + let first = t.chars().next().unwrap(); + if !(first.is_uppercase() || first.is_ascii_digit()) { + return false; + } + if t.ends_with(',') || t.ends_with(';') { + return false; + } + } + non_empty >= TABLE_MIN_COLUMNS } /// Try to extend a candidate table starting at `start_idx`. On success returns @@ -1076,7 +1401,35 @@ fn absorb_header_lines( let mut j = start_idx; while j > floor { let cand = j - 1; - let cells = split_cells(&lines[cand]); + let mut cells = split_cells(&lines[cand]); + // PDFium routinely emits a header row's words as one merged text run, + // so an 8-column table arrives with a 2-cell header whose second cell + // spans seven tracks. `match_track_idx` below can only bind that blob + // to a single column, collapsing the whole header band into one cell. + // Body rows already get merged-run recovery; headers need it too. + // + // Prose carries whitespace everywhere, so recovery "succeeds" on a + // paragraph sitting above the table just as readily as on a real + // header - manufacturing a header out of prose and dragging the + // paragraph into the table. Cell count can't separate them (a projected + // prose line splits on its own internal gaps), but length can: header + // labels are terse, shredded prose is not. + // + // Restricted to the first candidate (the line directly above the body): + // a merged-run header is a single line, whereas letting recovery + // track-align line after line lets absorption climb an entire + // paragraph, one plausible-looking row at a time. + if absorbed.is_empty() && cells.len() < column_count { + const HEADER_MAX_CELL_CHARS: usize = 30; + let tracks: Vec = track_ranges.iter().map(|r| r.0).collect(); + if let Some(recovered) = recover_merged_cell(cells.clone(), &tracks) + && recovered + .iter() + .all(|c| c.text.chars().count() <= HEADER_MAX_CELL_CHARS) + { + cells = recovered; + } + } if dbgt { let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect(); eprintln!( @@ -1135,6 +1488,116 @@ pub(super) fn detect_tables(lines: &[ProjectedLine]) -> Vec { detect_tables_impl(lines, true) } +/// `detect_tables` plus the booktabs rule-band pass: a table drawn as nothing +/// but a top and bottom hairline has no grid for the ruled detector and, when +/// it has only two columns, is below `TABLE_MIN_COLUMNS` for the borderless +/// one. The rules themselves are the missing evidence - see `rule_bands`. +pub(super) fn detect_tables_banded( + lines: &[ProjectedLine], + segs: &RuleSegments, + page_height: f32, +) -> Vec { + let pieces = compute_line_pieces(lines); + let runs = detect_tables_with_pieces(lines, &pieces, true); + let bands = rule_bands(segs, page_height); + if bands.is_empty() { + return runs; + } + two_col_band_pass(lines, &pieces, runs, &bands) +} + +/// A booktabs rule band: a pair of horizontal rules with nothing ruled between +/// them, i.e. the top and bottom of a table drawn without a grid. +/// +/// Requires the two rules to overlap in x (they bound the same block), to sit +/// far enough apart to hold rows but not so far as to be page furniture, and - +/// crucially - to have **no vertical rule crossing between them**. A band with +/// verticals is a real grid, and the ruled detector owns it. +fn rule_bands(segs: &RuleSegments, page_height: f32) -> Vec<(f32, f32)> { + let RuleSegments { hs, raw_vs: vs, .. } = segs; + let mut out = Vec::new(); + for pair in hs.windows(2) { + let (top, bot) = (&pair[0], &pair[1]); + let sep = bot.y - top.y; + if !(RULE_BAND_MIN_HEIGHT_PT..page_height * 0.5).contains(&sep) { + continue; + } + let overlap = (top.x_max.min(bot.x_max) - top.x_min.max(bot.x_min)).max(0.0); + let extent = (top.x_max - top.x_min).max(bot.x_max - bot.x_min).max(1.0); + if overlap / extent < 0.6 { + continue; + } + if vs + .iter() + .any(|v| v.y_min < bot.y - 1.0 && v.y_max > top.y + 1.0) + { + continue; + } + out.push((top.y, bot.y)); + } + out +} + +/// Last-resort pass for two-column tables enclosed in a rule band. +/// +/// Modelled on `two_row_second_pass` and gated the same way - only in the index +/// gaps where the normal pass found nothing, and only for runs that stay inside +/// the gap. The extra requirement is that the seed line lie inside a band and +/// that the band hold enough lines to be a table rather than a rule under a +/// heading or a hyperlink underline. +fn two_col_band_pass( + lines: &[ProjectedLine], + pieces: &[Vec>], + runs: Vec, + bands: &[(f32, f32)], +) -> Vec { + let claimed = |a: usize, b: usize| runs.iter().any(|r| r.start < b && a < r.end); + let mut extra: Vec = Vec::new(); + for &(y0, y1) in bands { + let inside: Vec = (0..lines.len()) + .filter(|&i| { + let c = lines[i].bbox.y + lines[i].bbox.height * 0.5; + c > y0 && c < y1 + }) + .collect(); + // The band's lines must be a contiguous run of the region and numerous + // enough to be a table body rather than a rule under a heading or a + // stack of hyperlink underlines. + if inside.len() < RULE_BAND_MIN_LINES + || inside.last().unwrap() - inside[0] + 1 != inside.len() + { + continue; + } + let (bs, be) = (inside[0], inside.last().unwrap() + 1); + if claimed(bs, be) { + continue; + } + // The seed must read as a genuine two-cell row, not a wrapped prose + // line that happens to sit under a rule. + if pieces[bs].len() != 2 { + continue; + } + // Detect against the band's lines alone: a run can then never reach + // past the rules that are the whole justification for relaxing + // `TABLE_MIN_COLUMNS` here. + if let Some(mut run) = + try_detect_table_inferred(&lines[bs..be], &pieces[bs..be], 0, 0, false, true) + { + run.start += bs; + run.end += bs; + run.body_start += bs; + extra.push(run); + } + } + if extra.is_empty() { + return runs; + } + let mut all = runs; + all.extend(extra); + all.sort_by_key(|r| r.start); + all +} + /// Count borderless table runs for the layout-complexity stats. Excludes /// description lists — a label/value pair block reads fine as text, so it /// should not flag the page as table-bearing. `GridFallback` runs count: @@ -1156,7 +1619,8 @@ pub(crate) fn validated_ruled_table_rects( page_width: f32, page_height: f32, ) -> Vec { - detect_ruled_tables_global(lines, graphics, page_width, page_height) + let segs = extract_rule_segments(graphics); + detect_ruled_tables_global(lines, &segs, page_width, page_height) .into_iter() .filter_map(|(_, consumed)| { let mut x0 = f32::INFINITY; @@ -1181,11 +1645,20 @@ pub(crate) fn validated_ruled_table_rects( } fn detect_tables_impl(lines: &[ProjectedLine], include_desc_lists: bool) -> Vec { + let pieces = compute_line_pieces(lines); + detect_tables_with_pieces(lines, &pieces, include_desc_lists) +} + +fn detect_tables_with_pieces( + lines: &[ProjectedLine], + pieces: &[Vec>], + include_desc_lists: bool, +) -> Vec { let mut out = Vec::new(); let mut i = 0; let mut floor = 0; while i < lines.len() { - if let Some(run) = try_detect_table_inferred(lines, i, floor) { + if let Some(run) = try_detect_table_inferred(lines, pieces, i, floor, false, false) { floor = run.end; i = run.end; out.push(run); @@ -1214,7 +1687,65 @@ fn detect_tables_impl(lines: &[ProjectedLine], include_desc_lists: bool) -> Vec< break; } } - merged + two_row_second_pass(lines, pieces, merged) +} + +/// Last-resort pass for header + single-data-row tables. +/// +/// Some real tables are a header row plus exactly one data row. Both length +/// gates in `try_detect_table_inferred` reject them. Relaxing those gates +/// *directly* is not safe: a 2-row run forms early, higher up the page, and +/// consumes the header of the real multi-row table below it. +/// +/// So instead we keep the normal pass exactly as-is and only retry, with the +/// relaxation on, inside the index gaps where it found *no* table at all. By +/// construction a run detected here cannot steal rows from a real table: there +/// isn't one in the gap. Runs that would spill past the gap end are discarded +/// for the same reason. +fn two_row_second_pass( + lines: &[ProjectedLine], + pieces: &[Vec>], + runs: Vec, +) -> Vec { + // Gaps between (and around) the runs the normal pass claimed. + let mut gaps: Vec<(usize, usize)> = Vec::new(); + let mut cursor = 0; + for r in &runs { + if r.start > cursor { + gaps.push((cursor, r.start)); + } + cursor = cursor.max(r.end); + } + if cursor < lines.len() { + gaps.push((cursor, lines.len())); + } + + let mut extra: Vec = Vec::new(); + for (gs, ge) in gaps { + let mut i = gs; + // `floor` starts at the gap start so header absorption can never walk + // back into the preceding table run. + let mut floor = gs; + while i < ge { + match try_detect_table_inferred(lines, pieces, i, floor, true, false) { + // A run that reaches past the gap is exactly the theft case the + // gap restriction exists to prevent. + Some(run) if run.end <= ge && run.start >= gs => { + floor = run.end; + i = run.end; + extra.push(run); + } + _ => i += 1, + } + } + } + if extra.is_empty() { + return runs; + } + let mut all = runs; + all.extend(extra); + all.sort_by_key(|r| r.start); + all } // ── Description-list 2-column table detector ────────────────────────────── @@ -2460,6 +2991,40 @@ const TABLE_MAX_PAGE_COVERAGE: f32 = 0.95; /// most of the table width. const RULED_HLINE_MIN_COVERAGE: f32 = 0.5; +/// Minimum fraction of the component's row extent a vertical rule must span to +/// count as a column boundary, the mirror image of `RULED_HLINE_MIN_COVERAGE`. +/// +/// Slide decks routinely draw a table as one stroked rect per cell, so a single +/// decorative highlight box behind a phrase *inside* a cell contributes a pair +/// of vertical edges that split that column in two for the whole table. +/// `collapse_gutter_columns` can't fuse the sliver back because text centres do +/// land inside it. +/// +/// Kept well below the horizontal counterpart: a genuine divider that rules only +/// the header band of an otherwise open table is a real layout, and must survive. +const RULED_VLINE_MIN_COVERAGE: f32 = 0.2; + +/// H/V rule segments extracted once from a page's graphics and shared by every +/// table-detection pass on the page. The global ruled pass, the per-region +/// ruled pass, the leaf-veto re-check, and the rule-band pass all consume the +/// same segments; extracting per call would redo the work once per region. +pub(super) struct RuleSegments { + /// Horizontal segments, clustered by y. + hs: Vec, + /// Vertical segments as extracted, before same-x clustering. The component + /// splitter needs these: clustering unions y-ranges across gaps. + raw_vs: Vec, + /// Vertical segments, clustered by x. + vs: Vec, +} + +pub(super) fn extract_rule_segments(graphics: &[GraphicPrimitive]) -> RuleSegments { + let (hs, raw_vs) = extract_h_v_segments(graphics); + let hs = cluster_h_segments(hs); + let vs = cluster_v_segments(raw_vs.clone()); + RuleSegments { hs, raw_vs, vs } +} + /// Extract horizontal and vertical line segments from a page's graphics. Each /// `Stroke` becomes one HSeg or VSeg depending on orientation; each stroked /// `Rect` contributes its four edges (cell-border rects, table frames). @@ -2524,23 +3089,44 @@ fn extract_h_v_segments(graphics: &[GraphicPrimitive]) -> (Vec, Vec) (hs, vs) } -/// Cluster H segments sharing a y-coordinate (within `TABLE_GRID_CLUSTER_PT`) -/// into a single wider grid line whose x-extent is the union of the inputs. +fn ranges_overlap_or_nearly_touch(a_min: f32, a_max: f32, b_min: f32, b_max: f32) -> bool { + a_min <= b_max + TABLE_CROSS_TOLERANCE_PT && b_min <= a_max + TABLE_CROSS_TOLERANCE_PT +} + +/// Cluster connected H segments sharing a y-coordinate (within +/// `TABLE_GRID_CLUSTER_PT`) into a single wider grid line whose x-extent is +/// the union of the inputs. Collinear segments separated by whitespace stay +/// distinct so unrelated side-by-side tables do not become one component. fn cluster_h_segments(mut segs: Vec) -> Vec { if segs.is_empty() { return segs; } segs.sort_by(|a, b| a.y.total_cmp(&b.y)); let mut out: Vec = Vec::with_capacity(segs.len()); - for seg in segs { - if let Some(last) = out.last_mut() - && (last.y - seg.y).abs() <= TABLE_GRID_CLUSTER_PT - { - last.x_min = last.x_min.min(seg.x_min); - last.x_max = last.x_max.max(seg.x_max); - continue; + let mut band_start = 0; + while band_start < segs.len() { + let band_y = segs[band_start].y; + let mut band_end = band_start + 1; + while band_end < segs.len() && (segs[band_end].y - band_y).abs() <= TABLE_GRID_CLUSTER_PT { + band_end += 1; } - out.push(seg); + + let band = &mut segs[band_start..band_end]; + band.sort_by(|a, b| a.x_min.total_cmp(&b.x_min)); + let mut current = band[0]; + current.y = band_y; + for seg in &band[1..] { + if ranges_overlap_or_nearly_touch(current.x_min, current.x_max, seg.x_min, seg.x_max) { + current.x_min = current.x_min.min(seg.x_min); + current.x_max = current.x_max.max(seg.x_max); + } else { + out.push(current); + current = *seg; + current.y = band_y; + } + } + out.push(current); + band_start = band_end; } out } @@ -2681,6 +3267,15 @@ struct CellGrid { /// Per-row flag: the row contains an alpha-dominant multi-column span (a /// group-header label, as opposed to a mostly-digit merged data run). row_alpha_spanner: Vec, + /// Per-cell flag: the rules say this cell is vertically merged with the one + /// above it, so it is empty by construction. See `count_rowspan_cells`. + /// Travels with the row/column collapses so the density gate forgives the + /// empties that are actually here, not ones from a discarded row. + spanned: Vec>, + /// Fraction of raw spans that cross an interior column boundary. Recorded + /// so `build_ruled_table` can tell whether dropping short vertical rules + /// actually stopped the grid cutting through text. + straddle_frac: f32, } impl CellGrid { @@ -2691,6 +3286,8 @@ impl CellGrid { has_text: vec![vec![false; n_cols]; n_rows], repl: vec![vec![String::new(); n_cols]; n_rows], row_alpha_spanner: vec![false; n_rows], + spanned: vec![vec![false; n_cols]; n_rows], + straddle_frac: 0.0, } } @@ -2737,6 +3334,7 @@ impl CellGrid { self.has_text = filter_by(std::mem::take(&mut self.has_text), keep); self.repl = filter_by(std::mem::take(&mut self.repl), keep); self.row_alpha_spanner = filter_by(std::mem::take(&mut self.row_alpha_spanner), keep); + self.spanned = filter_by(std::mem::take(&mut self.spanned), keep); } /// Keep only columns `c` where `keep[c]`; the per-cell layers filter, the @@ -2758,6 +3356,10 @@ impl CellGrid { .into_iter() .map(|row| filter_by(row, keep)) .collect(); + self.spanned = std::mem::take(&mut self.spanned) + .into_iter() + .map(|row| filter_by(row, keep)) + .collect(); } /// Drop "phantom rows" produced by stacked thin border-strip rects (some @@ -2962,7 +3564,18 @@ fn assign_cells( // nearest each crossed boundary x (xs[k] is column k's left // boundary, which is exactly the split target). let covered: Vec = (c_lo..=c_hi).collect(); - if let Some(pieces) = split_span_at_anchors(span, &covered, xs) { + // Prefer real word geometry: each word lands in the ruled column + // its own centre falls in, which is exact. `split_span_at_anchors` + // has to guess an x from a character index, so on a proportional + // font it can cut a word or two off from the true boundary. + if let Some(by_word) = bucket_words_into_columns(span, xs) { + for (col, text) in by_word { + grid.push_text(row, col, &text); + if !line.all_bold { + grid.is_bold[row][col] = false; + } + } + } else if let Some(pieces) = split_span_at_anchors(span, &covered, xs) { for (k, piece) in pieces.iter().enumerate() { grid.push_text(row, c_lo + k, piece); if !line.all_bold { @@ -3004,6 +3617,7 @@ fn assign_cells( } return None; } + grid.straddle_frac = straddle_frac; Some((grid, consumed_indices)) } @@ -3155,7 +3769,11 @@ fn merge_stacked_header( } /// Density gate: a grid that is mostly empty cells is rejected unless it shows -/// strong table evidence. Three escape hatches keep real tables: +/// strong table evidence. Cells the rules say are vertically merged into the +/// cell above (`spanned`) don't count as empty at all - they are empty by +/// construction, and a rowspan label column ("1. Embodying sustainability +/// values" beside three competence rows) otherwise reads as a sparse grid and +/// dies here. Beyond that, three escape hatches keep real tables: /// - **col0 spine**: a filled, short-text first column (a label column). /// - **long-prose table**: a large (≥5×3) grid with a bold header band covering /// ≥3 columns and a dense (≥70%-fill) inner description column — a @@ -3169,6 +3787,59 @@ fn merge_stacked_header( /// /// Returns `true` to keep the table, `false` to reject it. /// +/// Mark the grid cells that are *vertically merged* with the cell above them, +/// read straight off the rules: a cell whose top boundary carries no horizontal +/// stroke across its own column is the continuation of a rowspan. +/// +/// This is the geometric explanation for legitimately-empty cells. Every +/// interior `ys` boundary exists because *some* horizontal segment sits there, +/// so a column only scores when that particular rule stops short of it - which +/// is exactly how a PDF draws a merged cell. A fully-ruled grid scores zero. +fn rowspan_mask( + hs: &[HSeg], + h_indices: &[usize], + vs: &[VSeg], + v_indices: &[usize], + xs: &[f32], + ys: &[f32], +) -> Vec> { + let tol = TABLE_GRID_CLUSTER_PT; + let mut mask = vec![vec![false; xs.len() - 1]; ys.len() - 1]; + for (r, &y) in ys[1..ys.len() - 1].iter().enumerate() { + // Row `r + 1` is the one below boundary `ys[r + 1]`. + let row = r + 1; + // Some vertical rule must run *through* the boundary. That is what + // distinguishes "the grid continues, this one cell is merged" from + // "the grid ended here" - a page-frame component whose stray rules + // cross unruled prose has no vertical continuing past them, and + // forgiving its empties would turn body text into a two-column table. + let grid_continues = v_indices + .iter() + .map(|&i| &vs[i]) + .any(|v| v.y_min <= y - tol && v.y_max >= y + tol); + if !grid_continues { + continue; + } + let at_y: Vec<&HSeg> = h_indices + .iter() + .map(|&i| &hs[i]) + .filter(|h| (h.y - y).abs() <= tol) + .collect(); + for c in 0..xs.len() - 1 { + // Test the column's *centre*, not containment of the whole column: + // `xs` comes from clustered and gutter-collapsed boundaries, so the + // outer ones can sit tens of points off the drawn border, and cell + // rules are inset by their padding. The centre is the one point + // that is unambiguously inside this cell. + let cx = (xs[c] + xs[c + 1]) * 0.5; + if !at_y.iter().any(|h| h.x_min <= cx && h.x_max >= cx) { + mask[row][c] = true; + } + } + } + mask +} + fn passes_density_gate( cells: &[Vec], cell_has_text: &[Vec], @@ -3176,13 +3847,25 @@ fn passes_density_gate( n_rows: usize, n_cols: usize, flattened: bool, + spanned: &[Vec], dbg: bool, ) -> bool { let total = n_rows * n_cols; - let empty_count = cell_has_text - .iter() - .flatten() - .filter(|filled| !**filled) + // A spanned cell only excuses itself when the cell it is merged into + // actually holds text. Walk up the run of merged cells to its head: an + // empty head means nothing was merged here, just an unruled hole - the + // shape a chart's vertical gridlines make, where forgiving the empties + // would turn a plot into a 27-column table. + let merged_into_text = |r: usize, c: usize| { + let mut i = r; + while i > 0 && spanned[i][c] { + i -= 1; + } + cell_has_text[i][c] + }; + let empty_count = (0..n_rows) + .flat_map(|r| (0..n_cols).map(move |c| (r, c))) + .filter(|&(r, c)| !cell_has_text[r][c] && !(spanned[r][c] && merged_into_text(r, c))) .count(); let empty_frac = (empty_count as f32) / (total as f32); if empty_frac <= TABLE_MAX_EMPTY_CELL_FRACTION { @@ -3246,23 +3929,267 @@ fn passes_density_gate( /// Build a `TableRun` for one ruled-grid component. Returns `None` if the /// resulting grid is too small (< 2 cols or < 2 rows), covers nearly the /// whole page (likely the page border), or is mostly empty cells. -fn build_ruled_table( +/// Fuse "gutter" column boundaries left behind by paired cell-border edges. +/// +/// A bordered cell contributes two vertical edges - its own right edge and the +/// next cell's left edge - separated by the table's inter-cell padding. +/// `TABLE_COL_BOUNDARY_CLUSTER_PT` fuses the tight pairs (4-6pt), but a +/// generously padded table puts 15-25pt between them, leaving a sliver column +/// between every real column. Those slivers double the column count, and +/// because `split_span_at_anchors` shreds text into them they are *not* empty, +/// so `collapse_phantom_cols` can't remove them - the grid then fails the +/// empty-cell gate and a perfectly good table is thrown away. +/// +/// Width alone can't identify them - plenty of real tables carry a genuinely +/// narrow column (a `#` or `No.` column beside a wide description). What marks +/// a sliver is that **no text center ever lands inside it**: text steps over a +/// gutter, but a narrow real column still holds its own values. So fuse a +/// column only when it holds no span center *and* is far narrower than the +/// columns that do hold text. Span centers are read straight off the raw spans, +/// before `split_span_at_anchors` runs, so shredded fragments can't disguise a +/// sliver as occupied. +/// +/// The width guard matters independently: a worksheet's blank answer column is +/// also centerless, but it is as wide as its neighbours and must survive. +fn collapse_gutter_columns(xs: &mut Vec, lines: &[ProjectedLine], dbg: bool) { + /// A sliver must be at most this fraction of the median *content-bearing* + /// column width. Blank-but-full-width cells (worksheets, forms) sit well + /// above it and are preserved. + const GUTTER_MAX_WIDTH_FRAC: f32 = 0.5; + + if xs.len() < 4 { + return; // fewer than 3 columns: nothing to fuse without destroying the table + } + let centers: Vec = lines + .iter() + .flat_map(|l| l.spans.iter()) + .filter(|s| !s.text.trim().is_empty()) + .map(|s| s.x + s.width * 0.5) + .collect(); + let has_center = |lo: f32, hi: f32| centers.iter().any(|&c| c >= lo && c < hi); + + let before = xs.len(); + while xs.len() > 3 { + // Median width of the columns that actually hold text. + let mut occupied: Vec = xs + .windows(2) + .filter(|w| has_center(w[0], w[1])) + .map(|w| w[1] - w[0]) + .collect(); + if occupied.is_empty() { + break; + } + occupied.sort_by(|a, b| a.total_cmp(b)); + let median = occupied[occupied.len() / 2]; + let max_sliver = median * GUTTER_MAX_WIDTH_FRAC; + + // Narrowest centerless column, if any is slim enough to be a gutter. + let victim = xs + .windows(2) + .enumerate() + .filter(|(_, w)| !has_center(w[0], w[1])) + .map(|(i, w)| (i, w[1] - w[0])) + .filter(|&(_, width)| width < max_sliver) + .min_by(|a, b| a.1.total_cmp(&b.1)); + let Some((i, _)) = victim else { break }; + xs[i] = (xs[i] + xs[i + 1]) * 0.5; + xs.remove(i + 1); + } + if dbg && xs.len() != before { + eprintln!( + "[ruled] gutter-collapse {before} -> {} boundaries", + xs.len() + ); + } +} + +/// Drop vertical rules too short to be column boundaries (see +/// `RULED_VLINE_MIN_COVERAGE`). Coverage is measured against the row extent the +/// horizontals describe, since that is the height a real divider has to reach. +fn filter_short_vlines( hs: &[HSeg], - vs: &[VSeg], h_indices: &[usize], + vs: &[VSeg], v_indices: &[usize], - lines: &[ProjectedLine], - page_width: f32, - page_height: f32, - pass: RuledPass, -) -> Option<(TableRun, Vec)> { - let dbg = *super::flags::DEBUG_RULED; - let mut xs: Vec = v_indices.iter().map(|&i| vs[i].x).collect(); - xs.sort_by(|a, b| a.total_cmp(b)); - // Coarser, mean-centered clustering for column boundaries: cell-border - // rects contribute paired edges 4-6pt apart that would otherwise become +) -> Vec { + let y_lo = h_indices.iter().map(|&i| hs[i].y).fold(f32::MAX, f32::min); + let y_hi = h_indices.iter().map(|&i| hs[i].y).fold(f32::MIN, f32::max); + let extent = (y_hi - y_lo).max(1.0); + let kept: Vec = v_indices + .iter() + .copied() + .filter(|&i| (vs[i].y_max - vs[i].y_min) / extent >= RULED_VLINE_MIN_COVERAGE) + .collect(); + // Two boundaries is one column, which is not a grid; below that the caller + // has nothing to refine and keeps the raw set. + if kept.len() >= 3 { + kept + } else { + v_indices.to_vec() + } +} + +/// Widen a boundary list to the extent the *perpendicular* rules imply. +/// +/// A table drawn with interior dividers but no outer frame gives `xs` that stop +/// at the first and last vertical rule, so its outermost columns are missing +/// entirely: every line in them reads as overhang and the component collapses. +/// The evidence is already on the page: the horizontal rules know how wide the +/// table is, and the verticals know how tall. +/// +/// Three guards, all load-bearing: +/// - **median, not min/max** of the perpendicular rules, so one overshooting +/// stroke cannot widen the grid; +/// - **the new outer band must hold text**, the same idiom +/// `collapse_gutter_columns` uses; otherwise a pen-cap overshoot or a +/// full-width section rule manufactures an empty outer column; +/// - **the band must be at least `min_band` wide/tall.** Callers pass the +/// smallest existing row height for the row axis: verticals routinely +/// overrun the last horizontal by a few points, and a band shorter than any +/// real row is that overshoot, not a row. A phantom row here is enough for +/// the ruled grid to outrank a better borderless table. The column axis +/// passes only the clustering tolerance, because an unruled *label* column +/// is both common and legitimately much narrower than the ruled data +/// columns it sits beside. +fn extend_to_perpendicular_extent( + axis: &mut Vec, + perp_los: &[f32], + perp_his: &[f32], + min_band: f32, + has_content: impl Fn(f32, f32) -> bool, +) { + let min_extension = min_band.max(TABLE_COL_BOUNDARY_CLUSTER_PT); + let median = |v: &[f32]| { + let mut s = v.to_vec(); + s.sort_by(f32::total_cmp); + s.get(s.len() / 2).copied() + }; + let (Some(lo), Some(hi)) = (median(perp_los), median(perp_his)) else { + return; + }; + if axis.is_empty() { + return; + } + let dbg = *super::flags::DEBUG_RULED; + if lo < axis[0] - min_extension && has_content(lo, axis[0]) { + if dbg { + eprintln!("[ruled] extend lo {:.1} -> {:.1}", axis[0], lo); + } + axis.insert(0, lo); + } + let last = axis[axis.len() - 1]; + if hi > last + min_extension && has_content(last, hi) { + if dbg { + eprintln!("[ruled] extend hi {last:.1} -> {hi:.1}"); + } + axis.push(hi); + } +} + +/// Build a ruled table from one grid component. +/// +/// Runs `build_ruled_table_from` twice when short vertical rules are present: +/// once on the rules as drawn, once with the stubs filtered out. The unfiltered +/// build is the gatekeeper: **the filtered result is only ever allowed to +/// replace a table that would have been produced anyway**. Filtering first +/// instead lets the filter *remove the evidence the gates reject junk on*: a +/// chart loses the very rules that made it too sparse or too small to pass, +/// and a rejected component flips into an accepted junk table. +fn build_ruled_table( + hs: &[HSeg], + vs: &[VSeg], + h_indices: &[usize], + v_indices: &[usize], + lines: &[ProjectedLine], + page_width: f32, + page_height: f32, + pass: RuledPass, +) -> Option<(TableRun, Vec)> { + let base = build_ruled_table_from( + hs, + vs, + h_indices, + v_indices, + lines, + page_width, + page_height, + pass, + )?; + let v_kept = filter_short_vlines(hs, h_indices, vs, v_indices); + let strip = |(run, consumed, _straddle)| (run, consumed); + if v_kept.len() == v_indices.len() { + return Some(strip(base)); + } + let dbg = *super::flags::DEBUG_RULED; + let retry = build_ruled_table_from( + hs, + vs, + h_indices, + &v_kept, + lines, + page_width, + page_height, + pass, + ); + // Take the refined grid only when dropping the stubs measurably stopped the + // boundaries cutting through text. A stub that sits on a *real* column edge + // is the common case in dense financial tables; there the straddle census + // is unchanged, and dropping a rule that was the only evidence for its + // boundary just merges a real column away. + match retry { + Some(r) if r.2 < base.2 - STRADDLE_IMPROVEMENT => { + if dbg { + eprintln!( + "[ruled] vline-coverage retry {} -> {} rules, straddle {:.2} -> {:.2}", + v_indices.len(), + v_kept.len(), + base.2, + r.2 + ); + } + Some(strip(r)) + } + _ => Some(strip(base)), + } +} + +/// How much the straddle fraction must fall for the short-vline refinement to +/// be worth taking. Removing a genuine phantom column drops it by an order of +/// magnitude; a stub on a real column edge leaves it flat. +const STRADDLE_IMPROVEMENT: f32 = 0.05; + +#[allow(clippy::too_many_arguments)] +fn build_ruled_table_from( + hs: &[HSeg], + vs: &[VSeg], + h_indices: &[usize], + v_kept: &[usize], + lines: &[ProjectedLine], + page_width: f32, + page_height: f32, + pass: RuledPass, +) -> Option<(TableRun, Vec, f32)> { + let dbg = *super::flags::DEBUG_RULED; + let mut xs: Vec = v_kept.iter().map(|&i| vs[i].x).collect(); + xs.sort_by(|a, b| a.total_cmp(b)); + let spans = || lines.iter().flat_map(|l| l.spans.iter()); + extend_to_perpendicular_extent( + &mut xs, + &h_indices.iter().map(|&i| hs[i].x_min).collect::>(), + &h_indices.iter().map(|&i| hs[i].x_max).collect::>(), + 0.0, + |lo, hi| { + spans().filter(|s| !s.text.trim().is_empty()).any(|s| { + let c = s.x + s.width * 0.5; + c >= lo && c < hi + }) + }, + ); + // Coarser, mean-centered clustering for column boundaries: cell-border + // rects contribute paired edges 4-6pt apart that would otherwise become // phantom 5pt "columns" the span splitter then shreds text into. cluster_boundaries(&mut xs, TABLE_COL_BOUNDARY_CLUSTER_PT); + collapse_gutter_columns(&mut xs, lines, dbg); // Distinct row y-coords (cluster again — multiple H lines may share a y). // In the global pass, first drop horizontal rules that span only a small @@ -3295,6 +4222,22 @@ fn build_ruled_table( } else { raw_ys(h_indices) }; + let mut ys = ys; + let min_row_band = ys.windows(2).map(|w| w[1] - w[0]).fold(f32::MAX, f32::min); + extend_to_perpendicular_extent( + &mut ys, + &v_kept.iter().map(|&i| vs[i].y_min).collect::>(), + &v_kept.iter().map(|&i| vs[i].y_max).collect::>(), + min_row_band, + |lo, hi| { + spans().filter(|s| !s.text.trim().is_empty()).any(|s| { + let c = s.y + s.height * 0.5; + c >= lo && c < hi + }) + }, + ); + dedup_close(&mut ys, TABLE_GRID_CLUSTER_PT); + let ys = ys; if dbg { eprintln!( "[ruled] component: ys={:?} xs={:?} ({} lines in scope)", @@ -3344,6 +4287,8 @@ fn build_ruled_table( // too many spans straddle interior column boundaries (decorative box, not // a table). let (mut grid, consumed_indices) = assign_cells(lines, &xs, &ys, dbg)?; + let straddle_frac = grid.straddle_frac; + grid.spanned = rowspan_mask(hs, h_indices, vs, v_kept, &xs, &ys); // Collapse phantom rows (text-less thin border-strip rects) then phantom // columns (text-less narrow border strips). A real table with one phantom @@ -3377,6 +4322,8 @@ fn build_ruled_table( has_text: cell_has_text, repl: cells_repl, row_alpha_spanner, + spanned, + straddle_frac: _, } = grid; let flattened_header = flatten_header_band( @@ -3400,6 +4347,18 @@ fn build_ruled_table( dbg, ); + // `merge_stacked_header` folds the top `k` rows into one, so realign the + // rowspan mask: the new first row is the merged header (never a rowspan + // continuation), and every later row keeps its own mask. + let spanned = if spanned.len() > n_rows { + let dropped = spanned.len() - n_rows; + let mut m = vec![vec![false; n_cols]; 1]; + m.extend(spanned[dropped + 1..].iter().cloned()); + m + } else { + spanned + }; + if !passes_density_gate( &cells, &cell_has_text, @@ -3407,6 +4366,7 @@ fn build_ruled_table( n_rows, n_cols, flattened_header.is_some(), + &spanned, dbg, ) { return None; @@ -3445,6 +4405,7 @@ fn build_ruled_table( }, }, consumed_indices, + straddle_frac, )) } @@ -3497,6 +4458,37 @@ fn dedup_close(v: &mut Vec, tol: f32) { /// Find the bucket index `i` such that `boundaries[i] <= val < boundaries[i+1]`. /// Returns `None` if `val` is outside the boundaries. +/// Distribute a multi-column run's words into ruled columns by each word's own +/// centre, returning `(column, text)` pairs in column order. +/// +/// This replaces guessing a split point from a character index: with word boxes +/// the answer is simply which cell each word sits in. Returns `None` when the +/// run has no verified word geometry, or when every word lands in one column - +/// in that case the caller's whole-span fallbacks are the honest answer, since +/// the run only *overlaps* the neighbouring cell (padding, an overhanging +/// descender) rather than genuinely spanning it. +fn bucket_words_into_columns(span: &TextItem, xs: &[f32]) -> Option> { + let words = verified_words(span)?; + let mut out: Vec<(usize, String)> = Vec::new(); + for w in words { + let cx = (w.x + w.width.max(0.0) * 0.5).clamp(xs[0], *xs.last()?); + let col = find_bucket(xs, cx)?; + match out.last_mut() { + Some((c, text)) if *c == col => { + text.push(' '); + text.push_str(w.text.trim()); + } + _ => out.push((col, w.text.trim().to_string())), + } + } + // Words must not zig-zag back into an earlier column, and the run must + // really occupy more than one. + if out.len() < 2 || out.windows(2).any(|p| p[1].0 <= p[0].0) { + return None; + } + Some(out) +} + fn find_bucket(boundaries: &[f32], val: f32) -> Option { if boundaries.len() < 2 || val < boundaries[0] || val > *boundaries.last().unwrap() { return None; @@ -3563,35 +4555,150 @@ pub fn detect_table_rects( out } +/// One band of a grid component after splitting, with the vertical rules +/// clipped to the band's own y-range. +struct GridBand { + h_idx: Vec, + v_idx: Vec, + /// A copy of the page's `vs` whose y-extents are clamped to this band, so + /// the shared spine rule cannot carry the neighbouring table's height into + /// `filter_short_vlines` or `extend_to_perpendicular_extent`. + vs: Vec, +} + +/// Split one grid component at row bands that no vertical rule actually spans. +/// +/// `cluster_v_segments` merges same-x verticals by taking the *union* of their +/// y-ranges with no gap check, so two tables stacked in one column, each +/// drawing its own short strokes at the same left edge, fuse into a single +/// component. Each table is then evaluated with all of the other's rows empty +/// and dies on `TABLE_MAX_EMPTY_CELL_FRACTION`, and when the two tables have +/// different layouts their column sets are unioned too, over-segmenting both. +/// +/// The cut signal is geometric and needs no threshold on emptiness: a band +/// between consecutive horizontals that **no raw, pre-cluster `VSeg` spans**. +/// That is strictly stronger than "a big gap", and it distinguishes this from a +/// rowspan, where the vertical does continue through the boundary. +fn split_component_at_grid_gaps( + hs: &[HSeg], + vs: &[VSeg], + raw_vs: &[VSeg], + h_idx: &[usize], + v_idx: &[usize], +) -> Vec { + /// Row pitch below this means the component is decoration, not a table - + /// vector-drawn maths glyphs make components with a 3-4pt pitch. + const MIN_PITCH_PT: f32 = 8.0; + /// A gap must dwarf the component's own row pitch *and* clear an absolute + /// floor, so ordinary row-height variation never cuts. + const GAP_PITCH_MULT: f32 = 2.5; + const GAP_MIN_PT: f32 = 30.0; + /// Each side must keep a real table: 3 boundaries is 2 rows, the minimum + /// `build_ruled_table` will look at. + const MIN_BAND_HLINES: usize = 3; + + let whole = || { + vec![GridBand { + h_idx: h_idx.to_vec(), + v_idx: v_idx.to_vec(), + vs: vs.to_vec(), + }] + }; + let mut sorted = h_idx.to_vec(); + sorted.sort_by(|&a, &b| hs[a].y.total_cmp(&hs[b].y)); + if sorted.len() < MIN_BAND_HLINES * 2 { + return whole(); + } + let mut pitches: Vec = sorted.windows(2).map(|w| hs[w[1]].y - hs[w[0]].y).collect(); + pitches.sort_by(f32::total_cmp); + let pitch = pitches[pitches.len() / 2]; + if pitch < MIN_PITCH_PT { + return whole(); + } + let min_gap = (pitch * GAP_PITCH_MULT).max(GAP_MIN_PT); + + let tol = TABLE_CROSS_TOLERANCE_PT; + let mut cuts: Vec = Vec::new(); // index into `sorted`: cut after this line + for (i, w) in sorted.windows(2).enumerate() { + let (lo, hi) = (hs[w[0]].y, hs[w[1]].y); + if hi - lo < min_gap { + continue; + } + let spanned = raw_vs + .iter() + .any(|v| v.y_min <= lo + tol && v.y_max >= hi - tol); + if !spanned { + cuts.push(i + 1); + } + } + if cuts.is_empty() { + return whole(); + } + + let mut bounds = vec![0usize]; + bounds.extend(cuts); + bounds.push(sorted.len()); + if bounds.windows(2).any(|b| b[1] - b[0] < MIN_BAND_HLINES) { + return whole(); // a sliver on one side: the cut is not describing two tables + } + + bounds + .windows(2) + .map(|b| { + let band = &sorted[b[0]..b[1]]; + let y_lo = hs[band[0]].y; + let y_hi = hs[band[band.len() - 1]].y; + let v_idx: Vec = v_idx + .iter() + .copied() + .filter(|&i| vs[i].y_max > y_lo + tol && vs[i].y_min < y_hi - tol) + .collect(); + let vs: Vec = vs + .iter() + .map(|v| VSeg { + x: v.x, + y_min: v.y_min.max(y_lo), + y_max: v.y_max.min(y_hi), + }) + .collect(); + GridBand { + h_idx: band.to_vec(), + v_idx, + vs, + } + }) + .collect() +} + /// Detect ruled-grid tables on a page from its vector graphics. Returns runs /// in document order (sorted by `start`). fn detect_ruled_tables_impl( lines: &[ProjectedLine], - graphics: &[GraphicPrimitive], + segs: &RuleSegments, page_width: f32, page_height: f32, pass: RuledPass, ) -> Vec<(TableRun, Vec)> { - let (hs, vs) = extract_h_v_segments(graphics); - let hs = cluster_h_segments(hs); - let vs = cluster_v_segments(vs); + let RuleSegments { hs, raw_vs, vs } = segs; if hs.len() < 2 || vs.len() < 2 { return Vec::new(); } - let components = find_grid_components(&hs, &vs); + let components = find_grid_components(hs, vs); let mut out = Vec::new(); for (h_idx, v_idx) in components { - if let Some(run) = build_ruled_table( - &hs, - &vs, - &h_idx, - &v_idx, - lines, - page_width, - page_height, - pass, - ) { - out.push(run); + for band in split_component_at_grid_gaps(hs, vs, raw_vs, &h_idx, &v_idx) { + if let Some(run) = build_ruled_table( + hs, + &band.vs, + &band.h_idx, + &band.v_idx, + lines, + page_width, + page_height, + pass, + ) { + out.push(run); + } } } out.sort_by_key(|(r, _)| r.start); @@ -3602,20 +4709,14 @@ fn detect_ruled_tables_impl( /// global pass needs. pub(super) fn detect_ruled_tables( lines: &[ProjectedLine], - graphics: &[GraphicPrimitive], + segs: &RuleSegments, page_width: f32, page_height: f32, ) -> Vec { - detect_ruled_tables_impl( - lines, - graphics, - page_width, - page_height, - RuledPass::PerRegion, - ) - .into_iter() - .map(|(r, _)| r) - .collect() + detect_ruled_tables_impl(lines, segs, page_width, page_height, RuledPass::PerRegion) + .into_iter() + .map(|(r, _)| r) + .collect() } /// Page-level ruled-table detection over *all* lines. A table whose rows @@ -3625,11 +4726,11 @@ pub(super) fn detect_ruled_tables( /// consumed so the caller can pull them out of the region pipeline. pub(super) fn detect_ruled_tables_global( lines: &[ProjectedLine], - graphics: &[GraphicPrimitive], + segs: &RuleSegments, page_width: f32, page_height: f32, ) -> Vec<(TableRun, Vec)> { - detect_ruled_tables_impl(lines, graphics, page_width, page_height, RuledPass::Global) + detect_ruled_tables_impl(lines, segs, page_width, page_height, RuledPass::Global) } /// Count filled (non-empty) cells in a TableRun. GridFallback returns 0 so @@ -3704,9 +4805,121 @@ pub(super) fn merge_table_runs( } } kept.sort_by_key(|r| r.start); + for run in &mut kept { + if let Block::Table { rows, .. } = &mut run.block { + merge_continuation_rows(rows); + } + } kept } +/// Fold soft-wrapped cell continuations back into the row they belong to. +/// +/// A table cell holding a sentence wraps across several projected lines, and +/// every one of those lines becomes its own grid row: +/// +/// ```text +/// | Knowledge | | ● To understand the meaning of reducing, reusing and recycling | +/// | | | and how they connect ● To understand the importance of the 3 Rs | +/// ``` +/// +/// As geometry this is unsolvable: the gap between a wrapped line and a genuine +/// next row is the same gap. But once the grid exists it is tractable from the +/// text alone. +/// +/// A row is a continuation of its predecessor when all of these hold: +/// +/// - it has the same width and an **empty first cell** (the row label lives on +/// the first line of the row and is never repeated); +/// - its filled columns are a **subset** of the predecessor's filled columns: +/// a continuation can only extend cells that already have text; +/// - it carries **no value-like cell**; numbers don't soft-wrap, so a +/// blank-label row holding one is a real data row (typically a `Total`); +/// - at least one extended cell reads as a soft wrap rather than a fresh +/// sentence after a completed one. +fn merge_continuation_rows(rows: &mut Vec>) { + if rows.len() < 2 { + return; + } + // The whole rule keys off "empty first cell", which only means "wrapped + // line" when the first column is a *label* column. Plenty of tables just + // have a sparse first column: a timetable whose `Notes` column is blank on + // every run, a size chart, a hierarchical header. There, every row looks + // like a continuation and the table collapses into a single row. Requiring + // the first column to be populated in at least half the rows separates the + // two. + let first_col_filled = rows + .iter() + .filter(|r| r.first().is_some_and(|c| !c.trim().is_empty())) + .count(); + if first_col_filled * 2 < rows.len() { + return; + } + let mut out: Vec> = Vec::with_capacity(rows.len()); + for row in rows.drain(..) { + if let Some(prev) = out.last_mut() { + if is_continuation_row(prev, &row) { + for (i, cell) in row.iter().enumerate() { + let t = cell.trim(); + if t.is_empty() { + continue; + } + // `prev[i]` is non-empty for every filled column of `row`; + // that is the subset rule `is_continuation_row` enforces. + prev[i].push(' '); + prev[i].push_str(t); + } + continue; + } + } + out.push(row); + } + *rows = out; +} + +fn is_continuation_row(prev: &[String], cur: &[String]) -> bool { + if cur.len() != prev.len() || cur.len() < 2 { + return false; + } + // The row label is never repeated on a wrapped line. + if !cur[0].trim().is_empty() || prev[0].trim().is_empty() { + return false; + } + let filled: Vec = (0..cur.len()) + .filter(|&i| !cur[i].trim().is_empty()) + .collect(); + if filled.is_empty() { + return false; + } + // A continuation extends existing text; it cannot introduce a new column. + if filled.iter().any(|&i| prev[i].trim().is_empty()) { + return false; + } + // Numbers never soft-wrap. A blank-label row carrying any value-like cell is + // a real data row - most often a `Total` line, whose label column is blank + // precisely because it isn't the row's own label. Vetoing on *any* value + // (not all of them) is what separates it from a genuine wrap, since such + // rows pair a short summary word with the figures. + if filled.iter().any(|&i| is_value_like(cur[i].trim())) { + return false; + } + filled + .iter() + .any(|&i| cell_continues(prev[i].trim_end(), cur[i].trim_start())) +} + +/// Whether `cur` reads as the tail of `prev` rather than a new statement. The +/// only shape we reject is a fresh capitalised sentence following a cell that +/// already closed with terminal punctuation - that is a sub-row, not a wrap. +fn cell_continues(prev: &str, cur: &str) -> bool { + let starts_new_sentence = cur + .chars() + .find(|c| c.is_alphabetic()) + .is_some_and(|c| c.is_uppercase()); + let prev_closed = prev.ends_with(['.', '!', '?']); + !(starts_new_sentence && prev_closed) +} + /// Escape `|` and `\n` inside a markdown table cell so the pipe-table grammar /// stays valid. Newlines should be impossible inside a single cell (we built /// cells from spans on the same projected line) but guard anyway. @@ -3721,6 +4934,229 @@ mod tests { use super::super::test_helpers::{line, line_with_spans, rect_borders, stroke}; use super::*; + /// One PDFium run whose words sit at the given (x, width) positions. + fn run_with_words(words: &[(&str, f32, f32)]) -> TextItem { + let x = words[0].1; + let last = words[words.len() - 1]; + TextItem { + text: words + .iter() + .map(|(t, _, _)| *t) + .collect::>() + .join(" "), + x, + width: last.1 + last.2 - x, + height: 10.0, + font_size: Some(10.0), + words: words + .iter() + .map(|(t, wx, ww)| crate::types::WordBox { + text: (*t).into(), + x: *wx, + y: 0.0, + width: *ww, + height: 10.0, + }) + .collect(), + ..Default::default() + } + } + + fn piece_texts(span: &TextItem) -> Vec { + split_span_at_gutters(span) + .into_iter() + .map(|p| p.text) + .collect() + } + + #[test] + fn merged_run_splits_at_its_column_gutters() { + // doc 190's header: PDFium emits eight cells as one run. In-cell word + // gaps are 1.76pt; the gutters are 7.7–12.2pt. + let span = run_with_words(&[ + ("Merge", 172.5, 18.3), + ("Method", 192.5, 21.9), + ("H6", 226.6, 8.6), + ("(Avg.)", 237.0, 18.1), + ("ARC", 263.5, 14.5), + ]); + assert_eq!(piece_texts(&span), ["Merge Method", "H6 (Avg.)", "ARC"]); + } + + #[test] + fn justified_prose_does_not_split() { + // Justification stretches spaces to 3.4pt against a 2.73pt norm - a + // 1.25× jump, far under the bimodality bar. This is the counterfeit the + // ratio test exists to reject. + let span = run_with_words(&[ + ("may", 306.1, 19.0), + ("not", 327.8, 14.1), + ("be", 344.6, 10.4), + ("as", 357.8, 9.2), + ("crucial.", 369.7, 32.7), + ("Thus,", 405.8, 24.8), + ("we", 433.3, 12.8), + ]); + assert_eq!(piece_texts(&span).len(), 1); + } + + #[test] + fn run_without_word_boxes_stays_whole() { + let mut span = run_with_words(&[("A", 0.0, 5.0), ("B", 20.0, 5.0), ("C", 26.0, 5.0)]); + span.words.clear(); + assert_eq!(piece_texts(&span), ["A B C"]); + } + + #[test] + fn kerning_outlier_gap_does_not_fake_bimodality() { + // One degenerate 0.1pt gap beside ordinary ~3.4pt word spaces. The + // raw ratio between them is huge, but 3.4pt in 10pt type is a word + // space, not a gutter tier; the font-relative floor must reject it. + let span = run_with_words(&[ + ("wide", 100.0, 20.0), + ("gap", 120.1, 15.0), + ("then", 138.5, 18.0), + ("normal", 159.9, 25.0), + ]); + assert_eq!(piece_texts(&span).len(), 1); + } + + #[test] + fn two_word_run_has_no_bimodality_to_measure() { + // One gap is trivially the largest; there is nothing to compare it to. + let span = run_with_words(&[("Added", 60.3, 24.1), ("Relative", 118.0, 31.1)]); + assert_eq!(piece_texts(&span), ["Added Relative"]); + } + + fn rows(v: &[&[&str]]) -> Vec> { + v.iter() + .map(|r| r.iter().map(|s| s.to_string()).collect()) + .collect() + } + + #[test] + fn wrapped_cell_lines_fold_into_their_row() { + let mut r = rows(&[ + &[ + "Knowledge", + "● To understand the meaning of reducing and recycling", + ], + &["", "and how they connect ● To be familiar with the 7 Rs"], + &["Skills", "● To implement waste management into daily"], + &["", "life ● To promote reducing before recycling"], + ]); + merge_continuation_rows(&mut r); + assert_eq!(r.len(), 2); + assert_eq!( + r[0][1], + "● To understand the meaning of reducing and recycling and how they connect ● To be familiar with the 7 Rs" + ); + assert_eq!(r[1][0], "Skills"); + } + + #[test] + fn totals_row_with_blank_label_is_not_a_continuation() { + // Regression: doc 45/47. A `Total` line legitimately has an empty label + // column; folding it into the last data row destroys both rows. + let mut r = rows(&[ + &[ + "7", + "Traditional and Modern Mental Health Organization", + "15", + ], + &["", "Total", "27,926"], + ]); + merge_continuation_rows(&mut r); + assert_eq!(r.len(), 2); + assert_eq!(r[1][1], "Total"); + } + + #[test] + fn sparse_first_column_disables_continuation_merging() { + // Regression: a timetable whose `Notes` column is blank on every run. + // Every row looks like a continuation, and merging collapses the whole + // table into one row (0.859 -> 0.006 GriTS on ParseBench). + let mut r = rows(&[ + &["", "8:24", "8:28", "8:41"], + &["", "8:29", "8:33", "8:46"], + &["", "8:33", "8:37", "8:50"], + &["", "8:38", "8:42", "8:55"], + ]); + merge_continuation_rows(&mut r); + assert_eq!(r.len(), 4); + } + + #[test] + fn continuation_may_not_introduce_a_new_column() { + let mut r = rows(&[&["Label", "some text", ""], &["", "more text", "brand new"]]); + merge_continuation_rows(&mut r); + assert_eq!(r.len(), 2, "col 2 was empty above, so this is a real row"); + } + + #[test] + fn fresh_sentence_after_closed_cell_is_not_a_continuation() { + let mut r = rows(&[ + &["Label", "A complete thought ending here."], + &["", "Another separate statement entirely."], + ]); + merge_continuation_rows(&mut r); + assert_eq!(r.len(), 2); + } + + #[test] + fn gutter_columns_fused_but_narrow_real_column_kept() { + // Padded cell borders: real columns 40..140, 160..260, 280..380 with + // 20pt gutters between them. Text sits inside the real columns only. + let rows = [ + line_with_spans( + &[("alpha", 45.0), ("beta", 165.0), ("gamma", 285.0)], + 100.0, + 10.0, + ), + line_with_spans( + &[("delta", 45.0), ("eps", 165.0), ("zeta", 285.0)], + 120.0, + 10.0, + ), + ]; + let mut xs = vec![40.0, 140.0, 160.0, 260.0, 280.0, 380.0]; + collapse_gutter_columns(&mut xs, &rows, false); + assert_eq!(xs.len(), 4, "two 20pt gutters should fuse: {xs:?}"); + + // A genuinely narrow but *occupied* column must survive, even though it + // is the same width as the gutters above. + let rows = [ + line_with_spans( + &[("1", 45.0), ("desc", 65.0), ("more text", 205.0)], + 100.0, + 10.0, + ), + line_with_spans( + &[("2", 45.0), ("desc two", 65.0), ("text here", 205.0)], + 120.0, + 10.0, + ), + ]; + let mut xs = vec![40.0, 60.0, 200.0, 380.0]; + let before = xs.clone(); + collapse_gutter_columns(&mut xs, &rows, false); + assert_eq!(xs, before, "occupied narrow column must not fuse"); + } + + #[test] + fn blank_worksheet_column_survives_gutter_collapse() { + // A worksheet's empty answer column is centerless like a gutter but is + // full width, so the width guard must keep it. + let rows = [ + line_with_spans(&[("K+", 45.0)], 100.0, 10.0), + line_with_spans(&[("Na+", 45.0)], 120.0, 10.0), + ]; + let mut xs = vec![40.0, 240.0, 440.0, 640.0]; + let before = xs.clone(); + collapse_gutter_columns(&mut xs, &rows, false); + assert_eq!(xs, before, "wide blank columns must not fuse: {xs:?}"); + } + #[test] fn split_cells_splits_on_wide_gaps() { let l = line_with_spans(&[("A", 50.0), ("B", 150.0), ("C", 250.0)], 100.0, 10.0); @@ -3731,6 +5167,226 @@ mod tests { assert_eq!(cells[2].text, "C"); } + fn hairline(y: f32, x: f32, w: f32) -> GraphicPrimitive { + GraphicPrimitive::Rect { + bbox: Rect { + x, + y, + width: w, + height: 0.8, + }, + stroke: Some("#000".to_string()), + fill: None, + } + } + + #[test] + fn booktabs_rule_pair_is_a_band() { + // doc 165: the whole table is two hairlines 98pt apart, no verticals. + let g = vec![hairline(139.5, 56.7, 225.9), hairline(237.4, 56.7, 225.9)]; + assert_eq!( + rule_bands(&extract_rule_segments(&g), 792.0), + [(139.5, 237.4)] + ); + } + + #[test] + fn ruled_grid_is_not_a_band() { + // Same rules, but a vertical runs between them: this is a real grid and + // the ruled detector owns it. Relaxing TABLE_MIN_COLUMNS here would be + // a second, worse opinion about the same table. + let mut g = vec![hairline(139.5, 56.7, 225.9), hairline(237.4, 56.7, 225.9)]; + g.push(GraphicPrimitive::Rect { + bbox: Rect { + x: 150.0, + y: 139.5, + width: 0.8, + height: 97.9, + }, + stroke: Some("#000".to_string()), + fill: None, + }); + assert!(rule_bands(&extract_rule_segments(&g), 792.0).is_empty()); + } + + #[test] + fn heading_underline_pair_is_not_a_band() { + // Two rules 6pt apart bound a heading underline, not a table body. + let g = vec![hairline(100.0, 56.7, 225.9), hairline(106.0, 56.7, 225.9)]; + assert!(rule_bands(&extract_rule_segments(&g), 792.0).is_empty()); + } + + /// Four full-height dividers plus one short pair from a highlight box drawn + /// inside a cell - the doc-200 shape. + fn vlines_with_intruder() -> (Vec, Vec) { + let hs = (0..5) + .map(|r| HSeg { + x_min: 0.0, + x_max: 300.0, + y: r as f32 * 25.0, + }) + .collect(); + let mut vs: Vec = [0.0, 100.0, 200.0, 300.0] + .iter() + .map(|&x| VSeg { + y_min: 0.0, + y_max: 100.0, + x, + }) + .collect(); + vs.push(VSeg { + y_min: 40.0, + y_max: 49.0, + x: 140.0, + }); + vs.push(VSeg { + y_min: 40.0, + y_max: 49.0, + x: 162.0, + }); + (hs, vs) + } + + #[test] + fn in_cell_highlight_box_does_not_become_a_column() { + let (hs, vs) = vlines_with_intruder(); + let kept = filter_short_vlines(&hs, &[0, 1, 2, 3, 4], &vs, &[0, 1, 2, 3, 4, 5]); + assert_eq!(kept, vec![0, 1, 2, 3]); + } + + #[test] + fn filter_never_leaves_fewer_than_three_columns() { + // Only two dividers reach full height; filtering would leave a 1-column + // grid, so the raw set survives and the usual gates decide. + let (hs, mut vs) = vlines_with_intruder(); + vs[2].y_min = 40.0; + vs[2].y_max = 49.0; + vs[3].y_min = 40.0; + vs[3].y_max = 49.0; + let kept = filter_short_vlines(&hs, &[0, 1, 2, 3, 4], &vs, &[0, 1, 2, 3, 4, 5]); + assert_eq!(kept, vec![0, 1, 2, 3, 4, 5]); + } + + #[test] + fn unruled_outer_column_is_recovered_from_the_horizontals() { + // Interior dividers at 124/383/652; the horizontals say the table runs + // 30..930, so the label column and the last data column are missing. + let mut xs = vec![123.8, 382.9, 652.1]; + extend_to_perpendicular_extent( + &mut xs, + &[29.7, 29.7, 29.8], + &[930.3, 930.3, 930.2], + 0.0, + |_, _| true, + ); + assert_eq!(xs, vec![29.7, 123.8, 382.9, 652.1, 930.3]); + } + + #[test] + fn outer_band_shorter_than_any_row_is_rule_overshoot() { + // Verticals overrun the last horizontal by 15.5pt on 26pt rows - a tail, + // not a row. + let mut ys = vec![181.4, 207.8, 234.2]; + extend_to_perpendicular_extent(&mut ys, &[181.4], &[249.7], 26.4, |_, _| true); + assert_eq!(ys, vec![181.4, 207.8, 234.2]); + } + + #[test] + fn empty_outer_band_is_not_a_column() { + let mut xs = vec![123.8, 382.9, 652.1]; + extend_to_perpendicular_extent(&mut xs, &[29.7], &[930.3], 0.0, |_, _| false); + assert_eq!(xs, vec![123.8, 382.9, 652.1]); + } + + #[test] + fn one_overshooting_rule_does_not_widen_the_grid() { + // Median, not min: a single stroke reaching to 29.7 is noise. + let mut xs = vec![123.8, 382.9, 652.1]; + extend_to_perpendicular_extent( + &mut xs, + &[29.7, 124.0, 123.9], + &[652.0, 652.2, 930.3], + 0.0, + |_, _| true, + ); + assert_eq!(xs, vec![123.8, 382.9, 652.1]); + } + + #[test] + fn rowspan_mask_marks_unruled_cell_boundaries() { + // Two columns; the boundary at y=20 is ruled only across column 1, so + // column 0's cell there is merged with the one above it. + let hs = vec![ + HSeg { + x_min: 0.0, + x_max: 100.0, + y: 0.0, + }, + HSeg { + x_min: 50.0, + x_max: 100.0, + y: 20.0, + }, + HSeg { + x_min: 0.0, + x_max: 100.0, + y: 40.0, + }, + ]; + let vs = vec![VSeg { + y_min: 0.0, + y_max: 40.0, + x: 50.0, + }]; + let mask = rowspan_mask( + &hs, + &[0, 1, 2], + &vs, + &[0], + &[0.0, 50.0, 100.0], + &[0.0, 20.0, 40.0], + ); + assert_eq!(mask, vec![vec![false, false], vec![true, false]]); + } + + #[test] + fn rowspan_mask_ignores_boundary_where_grid_ends() { + // No vertical crosses y=20, so the grid simply stops there - nothing is + // merged, and forgiving these empties would let a page frame shred + // prose into columns. + let hs = vec![ + HSeg { + x_min: 0.0, + x_max: 10.0, + y: 0.0, + }, + HSeg { + x_min: 0.0, + x_max: 10.0, + y: 20.0, + }, + HSeg { + x_min: 0.0, + x_max: 10.0, + y: 40.0, + }, + ]; + let vs = vec![VSeg { + y_min: 0.0, + y_max: 15.0, + x: 50.0, + }]; + let mask = rowspan_mask( + &hs, + &[0, 1, 2], + &vs, + &[0], + &[0.0, 50.0, 100.0], + &[0.0, 20.0, 40.0], + ); + assert!(mask.iter().flatten().all(|m| !*m)); + } + #[test] fn recover_merged_cell_splits_off_by_one() { // Mimics the page-6 case: row 0 establishes 3 tracks at 50/150/250. @@ -3932,7 +5588,7 @@ mod tests { line("d", 190.0, 155.0, 10.0, 10.0), // row 1, col 1 ]; - let runs = detect_ruled_tables(&lines, &graphics, 612.0, 792.0); + let runs = detect_ruled_tables(&lines, &extract_rule_segments(&graphics), 612.0, 792.0); assert_eq!(runs.len(), 1, "expected 1 ruled table, got {runs:?}"); match &runs[0].block { Block::Table { header, rows } => { @@ -3962,17 +5618,45 @@ mod tests { line("c", 90.0, 155.0, 10.0, 10.0), line("d", 190.0, 155.0, 10.0, 10.0), ]; - let runs = detect_ruled_tables(&lines, &graphics, 612.0, 792.0); + let runs = detect_ruled_tables(&lines, &extract_rule_segments(&graphics), 612.0, 792.0); assert_eq!(runs.len(), 1); } + #[test] + fn ruled_grids_with_shared_rows_remain_separate_components() { + // Two side-by-side 2x2 grids intentionally reuse the same y + // coordinates. Their horizontal borders must not be extended through + // the whitespace between the grids. + let mut graphics = Vec::new(); + for left in [50.0_f32, 350.0] { + for y in [100.0_f32, 140.0, 180.0] { + graphics.push(stroke(left, y, left + 200.0, y, 0.5)); + } + for x in [left, left + 100.0, left + 200.0] { + graphics.push(stroke(x, 100.0, x, 180.0, 0.5)); + } + } + + let (hs, vs) = extract_h_v_segments(&graphics); + let hs = cluster_h_segments(hs); + let vs = cluster_v_segments(vs); + let components = find_grid_components(&hs, &vs); + + assert_eq!(components.len(), 2, "expected two separate grid components"); + assert!( + components + .iter() + .all(|(h_indices, v_indices)| h_indices.len() == 3 && v_indices.len() == 3) + ); + } + #[test] fn ruled_table_page_border_rejected() { // Single big rect covering ~the whole page → should NOT be treated as a // table even though it has H+V lines on all four sides. let graphics = rect_borders(10.0, 10.0, 590.0, 770.0); let lines = vec![line("body text", 50.0, 400.0, 10.0, 10.0)]; - let runs = detect_ruled_tables(&lines, &graphics, 612.0, 792.0); + let runs = detect_ruled_tables(&lines, &extract_rule_segments(&graphics), 612.0, 792.0); assert!( runs.is_empty(), "page-border rect should not become a table, got {runs:?}" @@ -3990,7 +5674,7 @@ mod tests { graphics.push(stroke(x, 100.0, x, 190.0, 0.5)); } let lines = vec![line("only", 90.0, 115.0, 10.0, 10.0)]; - let runs = detect_ruled_tables(&lines, &graphics, 612.0, 792.0); + let runs = detect_ruled_tables(&lines, &extract_rule_segments(&graphics), 612.0, 792.0); assert!(runs.is_empty()); } @@ -4014,7 +5698,7 @@ mod tests { line("alice", 90.0, 155.0, 10.0, 10.0), line("99", 190.0, 155.0, 10.0, 10.0), ]; - let runs = detect_ruled_tables(&lines, &graphics, 612.0, 792.0); + let runs = detect_ruled_tables(&lines, &extract_rule_segments(&graphics), 612.0, 792.0); assert_eq!(runs.len(), 1); match &runs[0].block { Block::Table { header, rows } => { diff --git a/crates/liteparse/src/markdown_layout/test_helpers.rs b/crates/liteparse/src/markdown_layout/test_helpers.rs index e350f25d..9f6874ba 100644 --- a/crates/liteparse/src/markdown_layout/test_helpers.rs +++ b/crates/liteparse/src/markdown_layout/test_helpers.rs @@ -5,6 +5,7 @@ use crate::types::{Anchor, GraphicPrimitive, ParsedPage, ProjectedLine, Rect, Te pub(crate) fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine { ProjectedLine { text: text.into(), + rtl: crate::bidi::is_rtl_text(text), bbox: Rect { x, y, @@ -84,6 +85,7 @@ pub(crate) fn line_with_spans(cells: &[(&str, f32)], y: f32, size: f32) -> Proje .map(|(t, _)| *t) .collect::>() .join(" "), + rtl: false, bbox: Rect { x: min_x, y, @@ -134,6 +136,7 @@ pub(crate) fn styled_line(spans: &[(&str, f32, Option<&str>)], y: f32, size: f32 .map(|s| s.x + s.width) .fold(f32::NEG_INFINITY, f32::max); ProjectedLine { + rtl: crate::bidi::is_rtl_text(&joined), text: joined, bbox: Rect { x: min_x, diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs index bbafe130..a484d5a1 100644 --- a/crates/liteparse/src/ocr_merge.rs +++ b/crates/liteparse/src/ocr_merge.rs @@ -439,6 +439,7 @@ pub(crate) fn render_pages_for_ocr( dpi: f32, grayscale: bool, render_form_fields: bool, + continue_on_page_error: bool, ) -> Result, LiteParseError> { let mut rendered = Vec::new(); // With `render_form_fields`, draw form-field appearances into the OCR @@ -452,40 +453,53 @@ pub(crate) fn render_pages_for_ocr( form.run_document_actions(); } for (idx, page) in pages.iter().enumerate() { - let page_obj = document.page((page.page_number - 1) as i32)?; - let page_complexity = calculate_page_complexity(page, &page_obj)?; + let page_render = (|| -> Result, LiteParseError> { + let page_obj = document.page((page.page_number - 1) as i32)?; + let page_complexity = calculate_page_complexity(page, &page_obj)?; - if !page_complexity.needs_ocr { - continue; - } + if !page_complexity.needs_ocr { + return Ok(None); + } - // Clamp the render DPI so the long edge stays within the raster budget. - let long_edge_pt = page.page_width.max(page.page_height); - let mut eff_dpi = dpi; - if long_edge_pt > 0.0 { - let max_dpi = MAX_OCR_RENDER_LONG_EDGE_PX * 72.0 / long_edge_pt; - if eff_dpi > max_dpi { - eff_dpi = max_dpi; + // Clamp the render DPI so the long edge stays within the raster budget. + let long_edge_pt = page.page_width.max(page.page_height); + let mut eff_dpi = dpi; + if long_edge_pt > 0.0 { + let max_dpi = MAX_OCR_RENDER_LONG_EDGE_PX * 72.0 / long_edge_pt; + if eff_dpi > max_dpi { + eff_dpi = max_dpi; + } } - } - let bitmap = page_obj.render_with_form(eff_dpi, form.as_ref())?; - let width = bitmap.width() as u32; - let height = bitmap.height() as u32; - // Grayscale or RGB per the engine; see `OcrEngine::prefers_grayscale`. - let pixels = if grayscale { - bitmap.to_luma() - } else { - bitmap.to_rgb() - }; + let bitmap = page_obj.render_with_form(eff_dpi, form.as_ref())?; + let width = bitmap.width() as u32; + let height = bitmap.height() as u32; + // Grayscale or RGB per the engine; see `OcrEngine::prefers_grayscale`. + let pixels = if grayscale { + bitmap.to_luma() + } else { + bitmap.to_rgb() + }; - rendered.push(RenderedPage { - idx, - pixels, - width, - height, - dpi: eff_dpi, - }); + Ok(Some(RenderedPage { + idx, + pixels, + width, + height, + dpi: eff_dpi, + })) + })(); + match page_render { + Ok(Some(render)) => rendered.push(render), + Ok(None) => {} + // The page already extracted successfully, so a tolerant parse + // keeps its native text and only forgoes the OCR enrichment. + Err(error) if continue_on_page_error => eprintln!( + "[ocr] render failed for page {}: {} — keeping native text without OCR (continue_on_page_error)", + page.page_number, error + ), + Err(error) => return Err(error), + } } Ok(rendered) } diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rs index 01bd782f..554de1ab 100644 --- a/crates/liteparse/src/output/json.rs +++ b/crates/liteparse/src/output/json.rs @@ -66,8 +66,11 @@ pub(crate) struct JsonPage { #[derive(Debug, Serialize)] pub(crate) struct ParseResultJson { + pub total_pages: u32, pub pages: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] + pub page_errors: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] pub images: Vec, #[serde(skip_serializing_if = "is_zero")] pub image_error_count: u32, @@ -104,7 +107,9 @@ pub(crate) struct JsonImage { /// Build structured JSON output from parsed pages. pub(crate) fn build_json(pages: &[ParsedPage], extract_text_metadata: bool) -> ParseResultJson { ParseResultJson { + total_pages: pages.len().min(u32::MAX as usize) as u32, images: Vec::new(), + page_errors: Vec::new(), image_error_count: 0, form_type: None, xfa_packets: None, @@ -168,6 +173,7 @@ pub fn format_json_result( extract_text_metadata: bool, ) -> Result { let mut json = build_json(&result.pages, extract_text_metadata); + json.total_pages = result.total_pages; json.images = result .images .iter() @@ -184,6 +190,7 @@ pub fn format_json_result( duplicate_of: image.duplicate_of.clone(), }) .collect(); + json.page_errors = result.page_errors.clone(); json.image_error_count = result.image_error_count; json.form_type = result.form_type; json.xfa_packets = result.xfa_packets.clone(); @@ -387,14 +394,21 @@ mod tests { bytes: std::sync::Arc::new(vec![1, 2, 3]), }; let result = crate::parser::ParseResult { + total_pages: 3, pages: vec![], + page_errors: vec![crate::types::PageError { + page_number: 3, + message: "page extraction failed".into(), + }], text: String::new(), outline: vec![], images: vec![image], + screenshots: vec![], image_error_count: 2, form_type: None, creator: Some("LibreOffice".into()), producer: Some("LibreOffice 7.4".into()), + doc_meta: Some(crate::types::DocumentMetadata::default()), xfa_packets: Some(vec![crate::types::XfaPacket { index: 0, name: Some("datasets".into()), @@ -404,8 +418,10 @@ mod tests { }; let value: serde_json::Value = serde_json::from_str(&format_json_result(&result, false).unwrap()).unwrap(); + assert_eq!(value["total_pages"], 3); assert!(value.get("creator").is_none()); assert!(value.get("producer").is_none()); + assert!(value.get("doc_meta").is_none()); assert_eq!(value["xfa_packets"][0]["name"], "datasets"); assert_eq!(value["xfa_packets"][0]["content_length"], 11); assert_eq!(value["images"][0]["bbox"]["x"], 10.0); @@ -414,6 +430,8 @@ mod tests { assert_eq!(value["images"][0]["duplicate_of"], "p1_0"); assert!(value["images"][0].get("bytes").is_none()); assert_eq!(value["image_error_count"], 2); + assert_eq!(value["page_errors"][0]["page"], 3); + assert_eq!(value["page_errors"][0]["message"], "page extraction failed"); } #[test] diff --git a/crates/liteparse/src/output/markdown.rs b/crates/liteparse/src/output/markdown.rs index 8d698050..79899d00 100644 --- a/crates/liteparse/src/output/markdown.rs +++ b/crates/liteparse/src/output/markdown.rs @@ -134,6 +134,7 @@ mod tests { fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine { ProjectedLine { text: text.into(), + rtl: crate::bidi::is_rtl_text(text), bbox: Rect { x, y, diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs index 486f4188..ee2b869c 100644 --- a/crates/liteparse/src/parser.rs +++ b/crates/liteparse/src/parser.rs @@ -11,17 +11,23 @@ use crate::ocr::tesseract::TesseractOcrEngine; use crate::ocr_merge; use crate::output::markdown; use crate::projection; -#[cfg(not(target_arch = "wasm32"))] use crate::render; use crate::types::{ - ExtractedImage, OutlineTarget, Page, ParsedPage, PdfInput, ScreenshotRect, XfaPacket, + DocumentMetadata, ExtractedImage, OutlineTarget, Page, PageError, ParsedPage, PdfInput, + ScreenshotRect, XfaPacket, }; use pdfium::Library; /// Result of parsing a document. pub struct ParseResult { + /// Total number of pages in the source document, before `target_pages` or + /// `max_pages` limits are applied. + pub total_pages: u32, /// Parsed pages with projected text layout. pub pages: Vec, + /// Page-level PDFium extraction failures collected when + /// `continue_on_page_error` is enabled. + pub page_errors: Vec, /// Full document text, concatenated from all pages. pub text: String, /// Document outline (bookmarks) when present. Used by the markdown @@ -32,6 +38,9 @@ pub struct ParseResult { /// `id` and `format` the markdown emitter referenced, so the caller can /// match them up without parsing markdown. pub images: Vec, + /// Page screenshots encoded as PNG. Empty unless `extract_screenshots` + /// is enabled. + pub screenshots: Vec, /// Number of embedded image objects that could not be extracted. A bad /// image does not fail the rest of the document parse. pub image_error_count: u32, @@ -42,6 +51,11 @@ pub struct ParseResult { pub creator: Option, /// The document's `/Info` `Producer` entry, when present. pub producer: Option, + /// Document provenance metadata (dates, version/security, signatures, + /// incremental-save markers, trailer IDs, raw XMP, and source size). + /// Present only when `extract_document_metadata` is enabled, and `None` + /// for inputs converted from a non-PDF format. + pub doc_meta: Option, /// Raw XFA packets, present only when `extract_xfa_packets` is enabled. /// `Some([])` means extraction ran on a non-XFA document. pub xfa_packets: Option>, @@ -160,6 +174,33 @@ fn default_glyph_resolver() -> Option> None } +/// A document input already converted to PDF, if it needed converting. +/// +/// Holds the [`conversion::PdfInputGuard`] so the temporary file produced for +/// a DOCX/XLSX/PPTX/image source stays alive for as long as the resolved input +/// is usable. Reusing one of these across several parses is what keeps batch +/// parsing from re-running LibreOffice for every batch. +pub(crate) struct ResolvedInput { + input: PdfInput, + #[cfg(not(target_arch = "wasm32"))] + guard: conversion::PdfInputGuard, +} + +impl ResolvedInput { + /// True when `input` points at a temporary PDF we produced, so raw-file + /// provenance describes the intermediate rather than the caller's document. + fn is_converted(&self) -> bool { + #[cfg(not(target_arch = "wasm32"))] + { + self.guard.is_converted() + } + #[cfg(target_arch = "wasm32")] + { + false + } + } +} + /// Main LiteParse orchestrator. /// /// ### Thread safety @@ -175,6 +216,7 @@ fn default_glyph_resolver() -> Option> /// safe but their PDFium portions run sequentially. The OCR pass and grid /// projection (which dominate runtime for OCR-heavy documents) run outside /// the lock and remain fully concurrent. +#[derive(Clone)] pub struct LiteParse { config: LiteParseConfig, /// Optional caller-provided OCR engine. When set, this overrides the @@ -277,14 +319,24 @@ impl LiteParse { let lib = Library::init(); let document = extract::load_document_from_input(&lib, &validated_input, password)?; - let (pages, _, _) = extract::extract_pages_and_images( + // Complexity deliberately runs against the flattened document: once + // widget text lives in the content stream it is genuinely within + // PDFium's reach, so it should count toward the page's text budget + // instead of routing the page to OCR to recover text we already + // have. `AnnotationText` still fires for the non-widget appearance + // text it was introduced for. + let pages = extract::extract_pages_and_images( &document, target_pages.as_deref(), self.config.max_pages, false, // extract_links: irrelevant for complexity stats self.glyph_resolver.as_deref(), - extract::ExtractionOutputOptions::default(), - )?; + extract::ExtractionOutputOptions { + continue_on_page_error: self.config.continue_on_page_error, + ..Default::default() + }, + )? + .pages; let t_extract = web_time::Instant::now(); log(&format!( "[liteparse] extract: {:.1}ms ({} pages)", @@ -292,13 +344,29 @@ impl LiteParse { pages.len() )); - let page_complexities = pages - .iter() - .map(|page| { - let page_obj = document.page((page.page_number - 1) as i32)?; - ocr_merge::calculate_page_complexity(page, &page_obj) - }) - .collect::, _>>()?; + // In tolerant mode a page whose stats fail is dropped from both + // vectors (consumers match stats to pages by `page_number`), so + // the layout zip below stays aligned. + let mut kept_pages = Vec::with_capacity(pages.len()); + let mut page_complexities = Vec::with_capacity(pages.len()); + for page in pages { + let stats = document + .page((page.page_number - 1) as i32) + .map_err(LiteParseError::from) + .and_then(|page_obj| ocr_merge::calculate_page_complexity(&page, &page_obj)); + match stats { + Ok(stats) => { + page_complexities.push(stats); + kept_pages.push(page); + } + Err(error) if self.config.continue_on_page_error => log(&format!( + "[liteparse] complexity failed on page {}: {}", + page.page_number, error + )), + Err(error) => return Err(error), + } + } + let pages = kept_pages; log(&format!( "[liteparse] complexity: {:.1}ms", web_time::Instant::now() @@ -346,6 +414,54 @@ impl LiteParse { /// Use `PdfInput::Path` for files on disk or `PdfInput::Bytes` for /// in-memory PDF data (e.g. from a network response or Node.js Buffer). pub async fn parse_input(&self, input: PdfInput) -> Result { + self.validate_output_config()?; + let resolved = self.resolve_input(input).await?; + let target_pages = self.resolve_target_pages()?; + self.parse_resolved( + &resolved, + target_pages.as_deref(), + self.config.max_pages, + None, + ) + .await + } + + /// Convert a non-PDF input to PDF (if needed) and return it alongside the + /// guard that keeps any temporary file alive. + /// + /// Split out of [`LiteParse::parse_input`] so [`ParseSession`] can pay the + /// conversion cost once and reuse the result for every page batch. + async fn resolve_input(&self, input: PdfInput) -> Result { + #[cfg(not(target_arch = "wasm32"))] + { + let (input, guard) = + conversion::resolve_pdf_input(input, self.config.password.as_deref(), false) + .await?; + Ok(ResolvedInput { input, guard }) + } + #[cfg(target_arch = "wasm32")] + { + Ok(ResolvedInput { input }) + } + } + + /// Parse an already-resolved input over an explicit page selection. + /// + /// `target_pages` and `max_pages` are parameters rather than config reads + /// so batch parsing can narrow the selection per batch. + /// + /// `outline` lets a caller that already walked the bookmark tree supply it + /// instead of paying for it again. The walk resolves a page destination per + /// entry, which costs several times more than opening the document, so a + /// batch parse that recomputed it per batch would spend most of its + /// overhead there. + async fn parse_resolved( + &self, + resolved: &ResolvedInput, + target_pages: Option<&[u32]>, + max_pages: usize, + outline: Option>, + ) -> Result { let log = |msg: &str| { if !self.config.quiet { eprintln!("{}", msg); @@ -354,17 +470,11 @@ impl LiteParse { let t0 = web_time::Instant::now(); - self.validate_output_config()?; + // Provenance facts describe the file on disk, so they are meaningless + // for a PDF we generated ourselves from a DOCX/XLSX/image. + let want_doc_meta = self.config.extract_document_metadata && !resolved.is_converted(); - #[cfg(not(target_arch = "wasm32"))] - let (validated_input, _guard) = - conversion::resolve_pdf_input(input, self.config.password.as_deref(), false).await?; - - #[cfg(target_arch = "wasm32")] - let validated_input = input; - - // Determine which pages to extract - let target_pages = self.resolve_target_pages()?; + let validated_input = &resolved.input; // Extract text (and pre-render OCR pages in one PDF load when OCR is on). // The PDFium lock is acquired for this entire critical section and @@ -421,14 +531,18 @@ impl LiteParse { #[allow(unused_mut)] // mutated only by the native image-output writer let ( pages, + page_errors, + total_pages, ocr_rendered, outline, mut images, + screenshots, image_error_count, complexity, form_type, creator, producer, + doc_meta, xfa_packets, ) = { let lib = Library::init(); @@ -437,24 +551,33 @@ impl LiteParse { .config .extract_form_fields .then(|| { - crate::acroform_repair::repair_orphaned_widgets( - &lib, - &validated_input, - password, - ) + crate::acroform_repair::repair_orphaned_widgets(&lib, validated_input, password) }) .flatten(); #[cfg(not(target_arch = "wasm32"))] - let document_input = repaired_input.as_ref().unwrap_or(&validated_input); + let document_input = repaired_input.as_ref().unwrap_or(validated_input); #[cfg(target_arch = "wasm32")] - let document_input = &validated_input; + let document_input = validated_input; let document = extract::load_document_from_input(&lib, document_input, password)?; + let total_pages = document.page_count().max(0) as u32; let form_type = self .config .extract_form_fields .then(|| document.form_type()); let creator = document.meta_text("Creator"); let producer = document.meta_text("Producer"); + let doc_meta = want_doc_meta.then(|| { + // AcroForm repair rewrites the file, so provenance has to come + // from the original document; fall back if it no longer loads. + #[cfg(not(target_arch = "wasm32"))] + if repaired_input.is_some() + && let Ok(source) = + extract::load_document_from_input(&lib, validated_input, password) + { + return crate::document_metadata::extract(validated_input, &source); + } + crate::document_metadata::extract(validated_input, &document) + }); let xfa_packets = self.config.extract_xfa_packets.then(|| { document .xfa_packets() @@ -472,18 +595,23 @@ impl LiteParse { }) .collect::>() }); - let outline = extract::extract_outline(&document); - let (pages, images, image_error_count) = extract::extract_pages_and_images( + let outline = outline.unwrap_or_else(|| extract::extract_outline(&document)); + let extracted = extract::extract_pages_and_images( &document, - target_pages.as_deref(), - self.config.max_pages, + target_pages, + max_pages, self.config.extract_links && self.config.output_format == crate::config::OutputFormat::Markdown, self.glyph_resolver.as_deref(), extract::ExtractionOutputOptions { + continue_on_page_error: self.config.continue_on_page_error, extract_content_bounds: self.config.extract_content_bounds, extract_images: self.config.effective_extract_images(), - emit_word_boxes: self.config.emit_word_boxes, + // The markdown table detector splits PDFium's merged + // multi-cell runs on real word geometry, so it needs word + // boxes even when the caller didn't ask for them. + emit_word_boxes: self.config.emit_word_boxes + || self.config.output_format == crate::config::OutputFormat::Markdown, extract_text_metadata: self.config.extract_text_metadata, extract_vector_graphics: self.config.extract_vector_graphics, extract_annotations: self.config.extract_annotations, @@ -491,6 +619,30 @@ impl LiteParse { extract_structure_tree: self.config.extract_structure_tree, }, )?; + let extract::ExtractedPages { + pages, + page_errors, + images, + image_error_count, + flattened_form_widgets, + } = extracted; + // Reopening the input costs a full parse, so it is confined to the + // one consumer that genuinely needs live widget annotations: the + // opt-in form renderer, which initializes the form environment to + // run document actions and paint computed field appearances that + // have no appearance stream to flatten. + // + // Plain rendering (OCR rasters, screenshots) does not need it — + // flattening promotes the widget appearances into page content, + // so the raster is the same either way. Complexity likewise runs + // on the flattened document by design (see `is_complex`). + let needs_pristine_document = flattened_form_widgets + && (self.config.ocr_enabled || self.config.extract_screenshots) + && self.config.render_form_fields; + let pristine_document = needs_pristine_document + .then(|| extract::load_document_from_input(&lib, document_input, password)) + .transpose()?; + let analysis_document = pristine_document.as_ref().unwrap_or(&document); let t_extract = web_time::Instant::now(); log(&format!( "[liteparse] extract: {:.1}ms ({} pages)", @@ -499,11 +651,12 @@ impl LiteParse { )); let rendered = if self.config.ocr_enabled { let r = ocr_merge::render_pages_for_ocr( - &document, + analysis_document, &pages, self.config.dpi, ocr_grayscale, self.config.render_form_fields, + self.config.continue_on_page_error, )?; log(&format!( "[liteparse] ocr render: {:.1}ms ({} pages)", @@ -519,27 +672,69 @@ impl LiteParse { }; let complexity = if self.config.include_complexity { - pages + let mut complexity = Vec::with_capacity(pages.len()); + for page in &pages { + let stats = analysis_document + .page((page.page_number - 1) as i32) + .map_err(LiteParseError::from) + .and_then(|page_obj| ocr_merge::calculate_page_complexity(page, &page_obj)); + match stats { + Ok(stats) => complexity.push(stats), + // The page's text is already extracted; a tolerant + // parse keeps it and just leaves `complexity` unset + // (stats attach by page number below). + Err(error) if self.config.continue_on_page_error => log(&format!( + "[liteparse] complexity failed on page {}: {}", + page.page_number, error + )), + Err(error) => return Err(error), + } + } + complexity + } else { + Vec::new() + }; + let screenshots = if self.config.extract_screenshots { + let page_numbers = pages .iter() - .map(|page| { - let page_obj = document.page((page.page_number - 1) as i32)?; - ocr_merge::calculate_page_complexity(page, &page_obj) - }) - .collect::, _>>()? + .map(|page| page.page_number as u32) + .collect::>(); + render::render_document_pages( + analysis_document, + Some(&page_numbers), + self.config.dpi, + self.config.detect_screenshot_rects, + self.config.render_form_fields, + self.config.continue_on_page_error, + )? + .into_iter() + .map(|page| ScreenshotResult { + page_num: page.page_num, + width: page.width, + height: page.height, + image_bytes: page.png_bytes, + is_solid_fill: page.is_solid_fill, + rects: page.rects, + }) + .collect() } else { Vec::new() }; // `lib` is dropped here, releasing the PDFium lock. ( pages, + page_errors, + total_pages, rendered, outline, images, + screenshots, image_error_count, complexity, form_type, creator, producer, + doc_meta, xfa_packets, ) }; @@ -578,7 +773,14 @@ impl LiteParse { // Attach per-page complexity signals, including the layout signals // that need the projected page (same as `is_complex()` reports). - for (page, mut stats) in parsed_pages.iter_mut().zip(complexity) { + // Matched by page number, not position: a tolerant parse may have no + // stats for a page whose complexity pass failed. + let mut complexity = complexity.into_iter().peekable(); + for page in parsed_pages.iter_mut() { + let Some(mut stats) = complexity.next_if(|stats| stats.page_number == page.page_number) + else { + continue; + }; stats.layout = Some(ocr_merge::calculate_layout_complexity(page)); page.complexity = Some(stats); } @@ -627,14 +829,18 @@ impl LiteParse { } Ok(ParseResult { + total_pages, pages: parsed_pages, + page_errors, text: full_text, outline, images, + screenshots, image_error_count, form_type, creator, producer, + doc_meta, xfa_packets, }) } @@ -648,6 +854,7 @@ impl LiteParse { /// is fully synchronous. Used when an external extractor (e.g. with its /// own font-recovery pipeline) owns text extraction. pub fn parse_from_pages(&self, pages: Vec, outline: Vec) -> ParseResult { + let total_pages = pages.len().min(u32::MAX as usize) as u32; let mut parsed_pages = projection::project_pages_to_grid(pages); let full_text = if self.config.output_format == crate::config::OutputFormat::Markdown { @@ -671,14 +878,18 @@ impl LiteParse { }; ParseResult { + total_pages, pages: parsed_pages, + page_errors: Vec::new(), text: full_text, outline, images: Vec::new(), + screenshots: Vec::new(), image_error_count: 0, form_type: None, creator: None, producer: None, + doc_meta: None, xfa_packets: None, } } @@ -745,6 +956,118 @@ impl LiteParse { pub fn config(&self) -> &LiteParseConfig { &self.config } + + /// Open a document for bounded-memory batch parsing. + /// + /// Returns an error if `target_pages` is configured — an explicit page + /// selection and generated batch ranges would be ambiguous together. + pub async fn open_batch_session( + &self, + input: PdfInput, + batch_size: usize, + ) -> Result { + self.validate_output_config()?; + if self.config.target_pages.is_some() { + return Err(LiteParseError::Config( + "batch parsing cannot be combined with target_pages".to_string(), + )); + } + if batch_size == 0 { + return Err(LiteParseError::Config( + "batch size must be at least 1".to_string(), + )); + } + + let input = self.resolve_input(input).await?; + // One cheap open (a few ms even for a 100 MB file — PDFium maps the + // file and parses the xref rather than reading it) so the caller knows + // the page count before the first batch is parsed, and so the + // document-level bookmark walk is paid once instead of per batch. + let (total_pages, outline) = { + let lib = Library::init(); + let document = extract::load_document_from_input( + &lib, + &input.input, + self.config.password.as_deref(), + )?; + ( + document.page_count().max(0) as u32, + extract::extract_outline(&document), + ) + }; + + Ok(ParseSession { + page_limit: total_pages.min(self.config.max_pages.min(u32::MAX as usize) as u32), + parser: self.clone(), + input, + total_pages, + outline, + next_page: 1, + batch_size, + }) + } +} + +/// A document opened once and parsed in bounded page batches. +pub struct ParseSession { + parser: LiteParse, + input: ResolvedInput, + total_pages: u32, + /// Walked once at open. Document-level, so every batch reports the same + /// outline, and re-walking it per batch would dominate batching overhead. + outline: Vec, + /// Last source page this session will parse: `min(total_pages, max_pages)`. + page_limit: u32, + /// Next source page to parse, 1-based. + next_page: u32, + batch_size: usize, +} + +/// One batch of pages from a [`ParseSession`]. +pub struct ParseBatch { + /// First source page in this batch, 1-based. + pub start_page: u32, + /// Last source page in this batch, 1-based and inclusive. + pub end_page: u32, + /// The pages in `start_page..=end_page`, parsed as an ordinary result. + pub result: ParseResult, +} + +impl ParseSession { + /// Total pages in the source document, before `max_pages` or batching. + pub fn total_pages(&self) -> u32 { + self.total_pages + } + + /// Parse and return the next batch, or `None` once every page within + /// `max_pages` has been yielded. + pub async fn next_batch(&mut self) -> Result, LiteParseError> { + if self.next_page > self.page_limit { + return Ok(None); + } + let start_page = self.next_page; + let end_page = start_page + .saturating_add(self.batch_size.min(u32::MAX as usize) as u32 - 1) + .min(self.page_limit); + let targets: Vec = (start_page..=end_page).collect(); + + let result = self + .parser + .parse_resolved( + &self.input, + Some(&targets), + targets.len(), + Some(self.outline.clone()), + ) + .await?; + + self.next_page = end_page.saturating_add(1); + Ok(Some(ParseBatch { + start_page, + end_page, + result, + })) + } } #[cfg(test)] @@ -945,4 +1268,51 @@ mod tests { assert_eq!(pages[0].markdown, "intro\n\n![](img_p1_1.jpg)\n\noutro"); assert_eq!(full_text, pages[0].markdown); } + + /// A non-PDF source is converted exactly once, when the session opens: + /// the converted temporary PDF outlives every batch and is only removed + /// when the session is dropped. Lives here rather than in the integration + /// tests because it asserts against [`ResolvedInput`] internals. + #[tokio::test] + #[serial_test::serial] + async fn test_batch_session_owns_converted_source_for_its_lifetime() { + if std::env::var("SKIP_INTEGRATION_TESTS").as_deref() == Ok("yes") { + return; + } + let parser = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + quiet: true, + ..LiteParseConfig::default() + }); + let mut session = parser + .open_batch_session( + PdfInput::Path("../../integration_tests_data/sample3.doc".to_string()), + 1, + ) + .await + .expect("should convert and open a .doc"); + + assert!(session.input.is_converted()); + let converted_path = match &session.input.input { + PdfInput::Path(p) => p.clone(), + PdfInput::Bytes(_) => panic!("a converted .doc should resolve to a temp file path"), + }; + + let mut pages = 0; + while let Some(batch) = session.next_batch().await.expect("batch should parse") { + pages += batch.result.pages.len(); + assert!( + std::path::Path::new(&converted_path).exists(), + "converted temp PDF should outlive every batch — a missing file \ + would mean it was re-resolved or cleaned up per batch" + ); + } + assert_eq!(pages, 2); + + drop(session); + assert!( + !std::path::Path::new(&converted_path).exists(), + "dropping the session should clean up the converted temp PDF" + ); + } } diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs index 6c22e020..f2d75ca7 100644 --- a/crates/liteparse/src/projection.rs +++ b/crates/liteparse/src/projection.rs @@ -2889,6 +2889,7 @@ pub fn project_pages_to_grid(pages: Vec) -> Vec { page.page_width, page.page_height, &obstacles, + &table_rects, ); ParsedPage { page_number: page.page_number, @@ -4597,6 +4598,7 @@ pub(crate) fn build_projected_lines( page_width: f32, page_height: f32, figures: &[Rect], + table_rects: &[Rect], ) -> (Vec, Region) { if items.is_empty() { return (Vec::new(), Region::default()); @@ -4619,7 +4621,20 @@ pub(crate) fn build_projected_lines( .cloned() .collect(); - let mut out: Vec = Vec::new(); + // Ruled-table ownership per item, resolved once up front. The y-banding + // loop below consults it for every item, and rescanning `table_rects` + // there would make line construction quadratic in the size of a y-band. + let item_regions: Vec> = if table_rects.is_empty() { + Vec::new() + } else { + items + .iter() + .map(|item| table_region_for_item(table_rects, item)) + .collect() + }; + let region_of = |index: usize| item_regions.get(index).copied().flatten(); + + let mut out: Vec = Vec::new(); for (path, indices) in leaves { // Sort within the leaf by y, tie-break by x. `build_one_line` re-sorts // by x for left→right concatenation; the y-banding loop here only @@ -4636,6 +4651,8 @@ pub(crate) fn build_projected_lines( let mut current: Vec = Vec::new(); let mut current_y: f32 = 0.0; let mut current_h: f32 = 0.0; + let mut current_region: Option = None; + let mut current_unowned = false; // PDFium occasionally reports anomalously large item heights (e.g. // 56pt for a single-word run whose real glyph height is ~13pt) when // the font's bounding box / line-height is baked into the text-matrix @@ -4652,6 +4669,8 @@ pub(crate) fn build_projected_lines( current.push(idx); current_y = y; current_h = h; + current_region = region_of(idx); + current_unowned = current_region.is_none(); continue; } // Use the SMALLER of the two heights for the y-band tolerance — @@ -4672,32 +4691,47 @@ pub(crate) fn build_projected_lines( let height_mismatch = raw_h > Y_BAND_HEIGHT_CAP && raw_h > current_h * 2.0; let tol_factor = if height_mismatch { 0.3 } else { 0.5 }; let same = (y - current_y).abs() < current_h.min(h) * tol_factor; - if same { + let item_region = region_of(idx); + let crosses_independent_tables = item_region + .is_some_and(|next| current_region.is_some_and(|current| current != next)); + if same && !crosses_independent_tables { current.push(idx); current_y = current_y.min(y); current_h = current_h.max(h); + match item_region { + Some(region) => current_region = Some(region), + None => current_unowned = true, + } } else { - out.push(build_one_line( - items, - ¤t, - path.clone(), - &heading_excl_figures, - )); + out.push(TableOwnedLine { + line: build_one_line(items, ¤t, path.clone(), &heading_excl_figures), + region: if current_unowned { + None + } else { + current_region + }, + }); current = vec![idx]; current_y = y; current_h = h; + current_region = item_region; + current_unowned = item_region.is_none(); } } if !current.is_empty() { - out.push(build_one_line( - items, - ¤t, - path.clone(), - &heading_excl_figures, - )); + out.push(TableOwnedLine { + line: build_one_line(items, ¤t, path.clone(), &heading_excl_figures), + region: if current_unowned { + None + } else { + current_region + }, + }); } } + reorder_independent_table_lines(&mut out, table_rects); + // Normalize `indent_x` to be leaf-relative: subtract each leaf's minimum // line bbox.x from every line in that leaf. This way list-nesting and // paragraph-indent comparisons in `markdown_layout.rs` use offsets from @@ -4707,7 +4741,8 @@ pub(crate) fn build_projected_lines( { use std::collections::HashMap; let mut leaf_min: HashMap, f32> = HashMap::new(); - for line in &out { + for owned in &out { + let line = &owned.line; let e = leaf_min .entry(line.region_path.clone()) .or_insert(f32::INFINITY); @@ -4715,7 +4750,8 @@ pub(crate) fn build_projected_lines( *e = line.indent_x; } } - for line in &mut out { + for owned in &mut out { + let line = &mut owned.line; if let Some(min) = leaf_min.get(&line.region_path) && min.is_finite() { @@ -4727,7 +4763,173 @@ pub(crate) fn build_projected_lines( } } - (out, region) + (out.into_iter().map(|owned| owned.line).collect(), region) +} + +struct TableOwnedLine { + line: ProjectedLine, + region: Option, +} + +/// Return the ruled-table region that owns an item. Besides text inside the +/// grid, include a short label immediately above it; table captions and +/// section headings commonly sit just outside the top border. A label that +/// overlaps multiple tables remains page-spanning and is not assigned to +/// either one. +fn table_region_for_item(rects: &[Rect], item: &ProjectedTextItem) -> Option { + let item_left = item.orig_x; + let item_right = item.orig_x + item.orig_width; + let item_top = item.orig_y; + let item_bottom = item.orig_y + item.orig_height; + let nearby_above = |rect: &Rect| { + let gap = rect.y - item_bottom; + gap >= -TABLE_LABEL_OVERLAP_PT && gap <= item.orig_height.max(TABLE_LABEL_GAP_PT) + }; + + let mut matches = rects.iter().enumerate().filter_map(|(index, rect)| { + let overlap_x = (item_right.min(rect.x + rect.width) - item_left.max(rect.x)).max(0.0); + let horizontally_owned = item.orig_width > 0.0 && overlap_x / item.orig_width >= 0.5; + let inside_y = item_top <= rect.y + rect.height && item_bottom >= rect.y; + (horizontally_owned && (inside_y || nearby_above(rect))).then_some(index) + }); + + let first = matches.next()?; + matches.next().is_none().then_some(first) +} + +const TABLE_LABEL_GAP_PT: f32 = 12.0; +const TABLE_LABEL_OVERLAP_PT: f32 = 2.0; + +/// Same-y rows from side-by-side grids naturally alternate left/right. Group +/// lines by their owning grid so the table detector sees one complete table +/// at a time. +fn reorder_independent_table_lines(lines: &mut [TableOwnedLine], rects: &[Rect]) { + if rects.len() < 2 { + return; + } + + let Some(region_ranks) = table_region_ranks(rects) else { + return; + }; + + // Reorder one xy-cut leaf at a time. `markdown_layout` recovers regions by + // scanning for maximal runs of equal `region_path` (`classify.rs`), so + // carrying a line across a leaf boundary would shatter that leaf into + // phantom regions and misalign the per-region table runs keyed off it. + let mut start = 0; + while start < lines.len() { + let mut end = start + 1; + while end < lines.len() && lines[end].line.region_path == lines[start].line.region_path { + end += 1; + } + reorder_leaf_table_lines(&mut lines[start..end], ®ion_ranks); + start = end; + } +} + +/// Sort one leaf's table-owned lines into whole-table order. +fn reorder_leaf_table_lines(lines: &mut [TableOwnedLine], region_ranks: &[usize]) { + let Some(first) = lines.iter().position(|owned| owned.region.is_some()) else { + return; + }; + let last = lines + .iter() + .rposition(|owned| owned.region.is_some()) + .unwrap_or(first); + + let span = &mut lines[first..=last]; + if span.iter().any(|owned| owned.region.is_none()) { + return; + } + + span.sort_by(|left, right| { + let (Some(left_region), Some(right_region)) = (left.region, right.region) else { + return std::cmp::Ordering::Equal; + }; + region_ranks[left_region] + .cmp(®ion_ranks[right_region]) + .then(left.line.bbox.y.total_cmp(&right.line.bbox.y)) + .then(left.line.bbox.x.total_cmp(&right.line.bbox.x)) + }); +} + +/// Produce a stable page-reading rank for every table rectangle. Rectangles +/// sharing a common vertical interval form one side-by-side band and sort by +/// x; separate bands sort top-to-bottom. `None` means there is no side-by-side +/// relationship, so projection order does not need rewriting. +fn table_region_ranks(rects: &[Rect]) -> Option> { + struct Band { + top: f32, + common_top: f32, + common_bottom: f32, + regions: Vec, + } + + let mut by_y: Vec = (0..rects.len()).collect(); + by_y.sort_by(|&left, &right| { + rects[left] + .y + .total_cmp(&rects[right].y) + .then(rects[left].x.total_cmp(&rects[right].x)) + }); + let mut bands: Vec = Vec::new(); + for region in by_y { + let rect = &rects[region]; + let bottom = rect.y + rect.height; + if let Some(band) = bands + .iter_mut() + .find(|band| rect.y < band.common_bottom && bottom > band.common_top) + { + band.common_top = band.common_top.max(rect.y); + band.common_bottom = band.common_bottom.min(bottom); + band.regions.push(region); + } else { + bands.push(Band { + top: rect.y, + common_top: rect.y, + common_bottom: bottom, + regions: vec![region], + }); + } + } + + let has_side_by_side = bands.iter().any(|band| { + band.regions.iter().enumerate().any(|(position, &left)| { + band.regions[position + 1..].iter().any(|&right| { + rects[left].x + rects[left].width <= rects[right].x + || rects[right].x + rects[right].width <= rects[left].x + }) + }) + }); + if !has_side_by_side { + return None; + } + + bands.sort_by(|left, right| left.top.total_cmp(&right.top)); + let mut ordered = Vec::with_capacity(rects.len()); + for band in &mut bands { + band.regions.sort_by(|&left, &right| { + rects[left] + .x + .total_cmp(&rects[right].x) + .then(rects[left].y.total_cmp(&rects[right].y)) + }); + ordered.extend(band.regions.iter().copied()); + } + + let mut ranks = vec![0; rects.len()]; + for (rank, region) in ordered.into_iter().enumerate() { + ranks[region] = rank; + } + Some(ranks) +} + +#[cfg(test)] +fn regions_in_rank_order(rects: &[Rect]) -> Vec { + let ranks = table_region_ranks(rects).expect("side-by-side regions"); + let mut regions: Vec = (0..rects.len()).collect(); + regions.sort_by_key(|®ion| ranks[region]); + regions } fn build_one_line( @@ -4737,11 +4939,15 @@ fn build_one_line( figures: &[Rect], ) -> ProjectedLine { // Sort by x so concatenation reads left→right even if reading order had - // rotated insertions. + // rotated insertions. `spans` stays in this x-ascending order — the table + // cell splitter and the inline emphasis renderer both read geometry off it + // — while `text` below is joined in *reading* order, which for a + // right-to-left line runs the other way across the page. let mut sorted: Vec = idxs.to_vec(); sorted.sort_by(|a, b| items[*a].item.x.total_cmp(&items[*b].item.x)); - let mut text = String::new(); + // Item texts in x order; joined once the line's base direction is known. + let mut piece_texts: Vec<&str> = Vec::with_capacity(sorted.len()); let mut min_x = f32::INFINITY; let mut min_y = f32::INFINITY; let mut max_x = f32::NEG_INFINITY; @@ -4770,7 +4976,7 @@ fn build_one_line( let mut mcid: Option = None; let mut spans: Vec = Vec::with_capacity(sorted.len()); - for (pos, &i) in sorted.iter().enumerate() { + for &i in sorted.iter() { let proj = &items[i]; let it = &proj.item; // `handle_rotation_reading_order` zeroes `item.rotation` after it @@ -4783,13 +4989,10 @@ fn build_one_line( span.rotation = proj.orig_rotation; spans.push(span); - // Concatenate item text. Use existing num_spaces from projection only as + // Collect item text. Use existing num_spaces from projection only as // a hint — the markdown emitter re-collapses whitespace, so we just // ensure there's *some* separation between adjacent items. - if pos > 0 && !text.ends_with(' ') { - text.push(' '); - } - text.push_str(&it.text); + piece_texts.push(it.text.as_str()); min_x = min_x.min(it.x); min_y = min_y.min(it.y); @@ -4849,6 +5052,25 @@ fn build_one_line( } } + // Join the line in reading order. PDFium already hands back each item's + // characters in logical order, so a right-to-left line only needs its + // *items* walked right-to-left across the page — otherwise an invoice line + // like "المجموع الفرعي: 75.00 SAR" comes out as "SAR 75.00 المجموع الفرعي:", + // detaching every label from its value. Direction is decided from the + // assembled line, so a neutral-only line (pure digits) stays LTR and every + // left-to-right document takes exactly the path it took before. + let rtl = crate::bidi::is_rtl_pieces(piece_texts.iter().copied()); + if rtl { + piece_texts.reverse(); + } + let mut text = String::new(); + for piece in &piece_texts { + if !text.is_empty() && !text.ends_with(' ') { + text.push(' '); + } + text.push_str(piece); + } + // NOTE on tie-breaks: `max_by_key` over a HashMap returns the *last* max it // iterates, and HashMap iteration order is randomized per process. Ties on // char-weight would therefore pick a different winner every run, making the @@ -4947,6 +5169,7 @@ fn build_one_line( ProjectedLine { text, + rtl, bbox: bbox.clone(), anchor, // Real column detection is deferred (carry-forward in MARKDOWN_PROGRESS). @@ -5052,6 +5275,40 @@ mod tests { } } + fn table_rect(x: f32, y: f32, width: f32, height: f32) -> Rect { + Rect { + x, + y, + width, + height, + } + } + + #[test] + fn table_region_ranks_order_two_side_by_side_bands_top_to_bottom() { + let rects = [ + table_rect(400.0, 300.0, 180.0, 80.0), + table_rect(40.0, 100.0, 180.0, 80.0), + table_rect(40.0, 300.0, 180.0, 80.0), + table_rect(400.0, 100.0, 180.0, 80.0), + ]; + + assert_eq!(regions_in_rank_order(&rects), vec![1, 3, 2, 0]); + } + + #[test] + fn table_region_ranks_do_not_bridge_non_overlapping_outer_tables() { + // A overlaps B and B overlaps C, but A and C only touch. Treating + // overlap as a transitive relation would order A/C by x before B. + let rects = [ + table_rect(40.0, 0.0, 180.0, 100.0), + table_rect(400.0, 50.0, 180.0, 100.0), + table_rect(40.0, 100.0, 180.0, 100.0), + ]; + + assert_eq!(regions_in_rank_order(&rects), vec![0, 1, 2]); + } + #[test] fn xy_cut_finds_column_gutter_on_two_column_layout() { // Two columns, 50pt-wide gutter centered at x=300. Each column has @@ -5105,7 +5362,7 @@ mod tests { items.push(item_at("L", 50.0, y, 200.0, 10.0)); items.push(item_at("R", 350.0, y, 200.0, 10.0)); } - let (lines, _region) = build_projected_lines(&items, 612.0, 792.0, &[]); + let (lines, _region) = build_projected_lines(&items, 612.0, 792.0, &[], &[]); // Left col's 5 lines come first (path [0, ...]), then right col's 5 // (path [1, ...]). assert_eq!(lines.len(), 10); @@ -5138,7 +5395,7 @@ mod tests { items.push(item_at("left side text", 50.0, y, 200.0, 10.0)); items.push(item_at("right side text", 350.0, y, 200.0, 10.0)); } - let (lines, _region) = build_projected_lines(&items, 612.0, 792.0, &[]); + let (lines, _region) = build_projected_lines(&items, 612.0, 792.0, &[], &[]); // First line (in pre-order) must be the title. let first = lines.first().expect("at least one line"); assert!( diff --git a/crates/liteparse/src/render.rs b/crates/liteparse/src/render.rs index d8c961f9..43cb98de 100644 --- a/crates/liteparse/src/render.rs +++ b/crates/liteparse/src/render.rs @@ -46,15 +46,17 @@ pub fn render_pages_to_png( dpi, detect_rects, render_form_fields, + false, ) } -fn render_document_pages( +pub(crate) fn render_document_pages( document: &pdfium::Document, page_numbers: Option<&[u32]>, dpi: f32, detect_rects: bool, render_form_fields: bool, + continue_on_page_error: bool, ) -> Result, LiteParseError> { let page_count = document.page_count() as u32; let pages: Vec = match page_numbers { @@ -71,43 +73,55 @@ fn render_document_pages( let mut results = Vec::with_capacity(pages.len()); for page_num in pages { - if page_num < 1 || page_num > page_count { - return Err(LiteParseError::Other(format!( - "page {page_num} out of range (document has {page_count} pages)" - ))); - } + let page_render = (|| -> Result { + if page_num < 1 || page_num > page_count { + return Err(LiteParseError::Other(format!( + "page {page_num} out of range (document has {page_count} pages)" + ))); + } - let page = document.page((page_num - 1) as i32)?; - let bitmap = page.render_with_form(dpi, form.as_ref())?; - let width = bitmap.width() as u32; - let height = bitmap.height() as u32; - let rgba = bitmap.to_rgba(); - - let is_solid_fill = is_solid_fill_rgba(&rgba, width as usize, height as usize); - // A solid-fill page has no structure to find; skip the scan (this is - // also the extract binary's cheap blank-page short-circuit). - let rects = if detect_rects && !is_solid_fill { - find_solid_rects_rgba( - &rgba, - width as usize, - height as usize, - page.width(), - page.height(), - ) - } else { - Vec::new() - }; + let page = document.page((page_num - 1) as i32)?; + let bitmap = page.render_with_form(dpi, form.as_ref())?; + let width = bitmap.width() as u32; + let height = bitmap.height() as u32; + let rgba = bitmap.to_rgba(); + + let is_solid_fill = is_solid_fill_rgba(&rgba, width as usize, height as usize); + // A solid-fill page has no structure to find; skip the scan (this is + // also the extract binary's cheap blank-page short-circuit). + let rects = if detect_rects && !is_solid_fill { + find_solid_rects_rgba( + &rgba, + width as usize, + height as usize, + page.width(), + page.height(), + ) + } else { + Vec::new() + }; - let png_bytes = encode_png(&rgba, width, height)?; + let png_bytes = encode_png(&rgba, width, height)?; - results.push(RenderedPage { - page_num, - width, - height, - png_bytes, - is_solid_fill, - rects, - }); + Ok(RenderedPage { + page_num, + width, + height, + png_bytes, + is_solid_fill, + rects, + }) + })(); + + match page_render { + Ok(rendered) => results.push(rendered), + // The page's text is already extracted; a tolerant parse keeps + // it and just forgoes this page's screenshot. + Err(error) if continue_on_page_error => eprintln!( + "[render] page {page_num} failed: {error} — skipping its screenshot (continue_on_page_error)" + ), + Err(error) => return Err(error), + } } Ok(results) diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs index aacaee4f..26ee7534 100644 --- a/crates/liteparse/src/types.rs +++ b/crates/liteparse/src/types.rs @@ -10,6 +10,52 @@ pub enum PdfInput { Bytes(Vec), } +/// Document-level provenance metadata extracted from PDFium plus a bounded +/// streaming scan of the source PDF. Fields stay optional so malformed +/// metadata never prevents the document itself from being parsed. +#[derive(Debug, Clone, Default, Serialize)] +pub struct DocumentMetadata { + #[serde(skip_serializing_if = "Option::is_none")] + pub creation_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mod_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_encrypted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub security_handler_revision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub permissions: Option, + /// Literal `%%EOF` markers in the file. A rough incremental-update signal: + /// embedded PDF attachments and content-stream text inflate it. + #[serde(skip_serializing_if = "Option::is_none")] + pub eof_section_count: Option, + /// Literal `startxref` markers in the file, with the same caveat. + #[serde(skip_serializing_if = "Option::is_none")] + pub startxref_count: Option, + /// Whether the two halves of the last trailer `/ID` array differ, which + /// usually means the file was updated after creation. `None` when no + /// hex-string `/ID` pair was found in the last 1 MiB. + #[serde(skip_serializing_if = "Option::is_none")] + pub trailer_id_pair_differs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_file_size: Option, + /// The document catalog's `/Metadata` XMP packet, capped at 64 KiB. + /// `None` when the document has none, when it is too large to resolve + /// cheaply (see `extract_document_metadata`), or in WASM builds. + #[serde(skip_serializing_if = "Option::is_none")] + pub xmp: Option, + /// True when the catalog's XMP stream exceeded the 64 KiB cap and `xmp` + /// holds only its first 64 KiB. + #[serde(skip_serializing_if = "Option::is_none")] + pub xmp_truncated: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature_byte_range_reaches_eof: Option, +} + /// Represents a single text item extracted from a PDF page, /// including its content, position, size, rotation, and font metadata. #[derive(Debug, Clone, Default, Serialize)] @@ -198,6 +244,17 @@ pub struct Page { pub structure_tree: Option, } +/// A page that could not be extracted while tolerant page errors were enabled. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct PageError { + /// Source page number (1-indexed). Serialized as `page` to match the + /// sibling per-page fields in the JSON output (`pages[]`, `images[]`). + #[serde(rename = "page")] + pub page_number: u32, + /// Human-readable extraction failure. + pub message: String, +} + /// One PDF page annotation. Coordinates use the same top-left, 72-DPI /// viewport space as [`TextItem`]. #[derive(Debug, Clone, Serialize)] @@ -584,6 +641,11 @@ pub struct ProjectedLine { pub all_mono: bool, pub all_strike: bool, pub spans: Vec, + /// True when the line's base direction is right-to-left (strong RTL + /// characters outnumber strong LTR ones). `spans` is always in x-ascending + /// order; consumers that rebuild the line's *text* must walk it in reverse + /// when this is set, or labels detach from their values. + pub rtl: bool, /// Path from the page's region-tree root to the leaf containing this line. /// Equality means "same leaf"; prefix relationship means "one contains the /// other". Replaces the prior flat `column_id` scheme so nested layouts diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs index 15380d6b..81816901 100644 --- a/crates/liteparse/tests/integration_test.rs +++ b/crates/liteparse/tests/integration_test.rs @@ -1,5 +1,6 @@ use std::path::Path; +use liteparse::config::OutputFormat; use liteparse::conversion::convert_data_to_pdf; use liteparse::ocr_merge::ComplexityReason; use liteparse::types::PdfInput; @@ -38,6 +39,31 @@ async fn test_screenshot_pdf_integration() { assert!(!results[0].image_bytes.is_empty()); } +#[tokio::test] +#[serial] +async fn test_parse_can_return_screenshots() { + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + extract_screenshots: true, + ..LiteParseConfig::default() + }); + let parsed = lit + .parse("../../integration_tests_data/sample.pdf") + .await + .expect("Should parse and render PDF pages"); + + assert_eq!(parsed.screenshots.len(), parsed.pages.len()); + assert_eq!( + parsed.screenshots[0].page_num, + parsed.pages[0].page_number as u32 + ); + assert!( + parsed.screenshots[0] + .image_bytes + .starts_with(b"\x89PNG\r\n\x1a\n") + ); +} + #[tokio::test] async fn test_screenshot_rejects_text_file() { let dir = tempfile::tempdir().unwrap(); @@ -91,6 +117,124 @@ async fn test_parse_bytes_image_integration() { .await .expect("Should be able to parse"); assert_eq!(parsed.pages.len(), 1); + assert_eq!(parsed.total_pages, 1); +} + +#[tokio::test] +#[serial] +async fn test_total_pages_precedes_target_page_filtering() { + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + quiet: true, + target_pages: Some("2".into()), + ..LiteParseConfig::default() + }); + let parsed = lit + .parse("../../integration_tests_data/filled_acroform.pdf") + .await + .expect("Should be able to parse a selected page"); + + assert_eq!(parsed.total_pages, 3); + assert_eq!(parsed.pages.len(), 1); + assert_eq!(parsed.pages[0].page_number, 2); +} + +/// Batching must not change what any individual page parses to — only how many +/// pages are materialized at once. +#[tokio::test] +#[serial] +async fn test_batch_parse_matches_whole_document() { + let config = || LiteParseConfig { + ocr_enabled: false, + quiet: true, + ..LiteParseConfig::default() + }; + let path = "../../integration_tests_data/filled_acroform.pdf"; + + let whole = LiteParse::new(config()) + .parse(path) + .await + .expect("whole-document parse should succeed"); + + let mut session = LiteParse::new(config()) + .open_batch_session(PdfInput::Path(path.to_string()), 2) + .await + .expect("should open a session"); + assert_eq!(session.total_pages(), whole.total_pages); + + let mut batches = Vec::new(); + while let Some(batch) = session.next_batch().await.expect("batch should parse") { + batches.push(batch); + } + + // 3 pages at 2 per batch: [1-2], [3-3]. + assert_eq!(batches.len(), 2); + assert_eq!((batches[0].start_page, batches[0].end_page), (1, 2)); + assert_eq!((batches[1].start_page, batches[1].end_page), (3, 3)); + + let batched: Vec<_> = batches.iter().flat_map(|b| b.result.pages.iter()).collect(); + assert_eq!(batched.len(), whole.pages.len()); + for (got, want) in batched.iter().zip(&whole.pages) { + assert_eq!(got.page_number, want.page_number); + assert_eq!(got.text, want.text); + } + for batch in &batches { + assert_eq!(batch.result.total_pages, whole.total_pages); + } +} + +/// `max_pages` bounds the session the same way it bounds a whole parse, and a +/// batch size larger than the document collapses to a single batch. +#[tokio::test] +#[serial] +async fn test_batch_parse_respects_max_pages() { + let mut session = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + quiet: true, + max_pages: 2, + ..LiteParseConfig::default() + }) + .open_batch_session( + PdfInput::Path("../../integration_tests_data/filled_acroform.pdf".to_string()), + 100, + ) + .await + .expect("should open a session"); + + let first = session + .next_batch() + .await + .expect("batch should parse") + .expect("a first batch"); + assert_eq!(session.total_pages(), 3, "reports the source page count"); + assert_eq!((first.start_page, first.end_page), (1, 2)); + assert_eq!(first.result.pages.len(), 2); + assert!( + session.next_batch().await.expect("clean end").is_none(), + "max_pages should end the session before page 3" + ); +} + +/// An explicit page selection and generated batch ranges are ambiguous +/// together, so the combination is rejected up front. +#[tokio::test] +#[serial] +async fn test_batch_parse_rejects_target_pages() { + let opened = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + quiet: true, + target_pages: Some("1-2".into()), + ..LiteParseConfig::default() + }) + .open_batch_session( + PdfInput::Path("../../integration_tests_data/filled_acroform.pdf".to_string()), + 25, + ) + .await; + assert!( + opened.is_err(), + "target_pages + batching should be rejected" + ); } #[tokio::test] @@ -152,28 +296,62 @@ async fn test_parse_office_doc_integration() { #[tokio::test] #[serial] async fn test_parse_pdf_integration() { - let lit = LiteParse::new(LiteParseConfig::default()); + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + extract_document_metadata: true, + ..LiteParseConfig::default() + }); let parsed = lit .parse("../../integration_tests_data/sample.pdf") .await .expect("Should be able to parse"); assert_eq!(parsed.pages.len(), 1); + let doc_meta = parsed.doc_meta.expect("doc_meta requested"); + assert!(doc_meta.file_version.is_some()); + assert_eq!(doc_meta.is_encrypted, Some(false)); + assert!(doc_meta.raw_file_size.is_some_and(|size| size > 0)); + assert!(doc_meta.eof_section_count.is_some_and(|count| count > 0)); + assert_eq!(doc_meta.signature_count, Some(0)); +} + +/// Provenance is opt-in and stays absent on the default path. +#[tokio::test] +#[serial] +async fn test_doc_meta_absent_unless_requested() { + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + ..LiteParseConfig::default() + }); + let parsed = lit + .parse("../../integration_tests_data/sample.pdf") + .await + .expect("Should be able to parse"); + assert!(parsed.doc_meta.is_none()); } #[tokio::test] #[serial] async fn test_parse_bytes_pdf_integration() { let fixture_path = "../../integration_tests_data/sample.pdf"; - let lit = LiteParse::new(LiteParseConfig::default()); + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + extract_document_metadata: true, + ..LiteParseConfig::default() + }); let data = tokio::fs::read(fixture_path) .await .expect("Should be able to read file"); + let expected_size = data.len() as u64; let input = PdfInput::Bytes(data); let parsed = lit .parse_input(input) .await .expect("Should be able to parse"); assert_eq!(parsed.pages.len(), 1); + assert_eq!( + parsed.doc_meta.and_then(|meta| meta.raw_file_size), + Some(expected_size) + ); } /// Stress test: many concurrent `parse_input` calls on a multi-threaded @@ -252,3 +430,154 @@ async fn test_annotation_text_complexity_reason() { page.reasons ); } + +/// Filled AcroForm values are visible page content even though PDFium's text +/// API does not expose widget appearance streams until they are flattened. +#[tokio::test] +#[serial] +async fn test_filled_acroform_values_are_extracted_as_text() { + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + output_format: OutputFormat::Markdown, + ..Default::default() + }); + let parsed = lit + .parse("../../integration_tests_data/filled_acroform.pdf") + .await + .expect("filled form should parse"); + + // See scripts/generate_filled_acroform_fixture.py for what each widget + // is meant to exercise. + for (expected, case) in [ + ( + "ACROFORM-CUSTOMER-7319", + "painted directly by the appearance", + ), + ("2026-07-28", "painted through a nested form XObject"), + ("50.00", "painted by the appearance and the content stream"), + ] { + assert_eq!( + parsed.text.matches(expected).count(), + 1, + "visible form value should appear exactly once ({case}): {expected}" + ); + assert!( + parsed.pages[0] + .text_items + .iter() + .any(|item| item.text.contains(expected)), + "form value should be a positioned text item ({case}): {expected}" + ); + } + assert!( + !parsed.text.contains("DEFAULT-ONLY-SHOULD-NOT-APPEAR"), + "an unpainted default choice must not be treated as a filled value" + ); + assert!( + !parsed.text.contains("ANNOTATION-ONLY-SHOULD-NOT-APPEAR"), + "non-widget annotation appearances must not become page text" + ); + assert!( + !parsed.text.contains("HIDDEN-SHOULD-NOT-APPEAR"), + "a hidden widget is never rendered, so its value is not visible text" + ); + // Flattening replaces the page content under a widget rect with that + // widget's appearance, so page text drawn there is dropped unless it is + // put back. This label sits inside the `amount` rect and no appearance + // reproduces it. + assert_eq!( + parsed.text.matches("PREPRINTED-LABEL").count(), + 1, + "page text under a widget rect must survive flattening exactly once" + ); + assert!( + parsed.pages[0].form_fields.is_none(), + "default text extraction must not enable structured form metadata" + ); + // Page 3's only annotation paints its value through a nested form XObject. + // Nothing else on that page would trigger a flatten, so this fails unless + // the appearance walk descends into form objects. + assert!( + parsed.pages[2] + .text_items + .iter() + .any(|item| item.text.contains("NESTED-ONLY-VALUE")), + "a widget whose value is painted only through a nested form XObject \ + must still be detected and flattened" + ); + + let with_metadata = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + output_format: OutputFormat::Markdown, + extract_annotations: true, + extract_form_fields: true, + ..Default::default() + }) + .parse("../../integration_tests_data/filled_acroform.pdf") + .await + .expect("filled form should parse with structured metadata"); + + assert_eq!( + with_metadata.pages[0].annotations.as_ref().unwrap().len(), + 6 + ); + assert!( + with_metadata.pages[0] + .annotations + .as_ref() + .unwrap() + .iter() + .any(|annotation| { + annotation.subtype == "freetext" + && annotation.contents.as_deref() == Some("ANNOTATION-ONLY-SHOULD-NOT-APPEAR") + }), + "non-widget annotation metadata should remain available when requested" + ); + let fields = with_metadata.pages[0].form_fields.as_ref().unwrap(); + assert_eq!(fields.len(), 5); + assert!(fields.iter().any(|field| { + field.name.as_deref() == Some("customer_name") + && field.value.as_deref() == Some("ACROFORM-CUSTOMER-7319") + })); + assert!(fields.iter().any(|field| { + field.name.as_deref() == Some("default_only_choice") + && field.value.as_deref() == Some("DEFAULT-ONLY-SHOULD-NOT-APPEAR") + })); + assert_eq!( + with_metadata.pages[1].annotations.as_ref().unwrap().len(), + 1 + ); + let second_page_fields = with_metadata.pages[1].form_fields.as_ref().unwrap(); + assert_eq!(second_page_fields.len(), 1); + assert_eq!( + second_page_fields[0].name.as_deref(), + Some("complexity_sentinel") + ); + assert_eq!(second_page_fields[0].value.as_deref(), Some("OK")); + let third_page_fields = with_metadata.pages[2].form_fields.as_ref().unwrap(); + assert_eq!(third_page_fields.len(), 1); + assert_eq!(third_page_fields[0].name.as_deref(), Some("nested_only")); + + // Complexity sees the flattened text. `AnnotationText` means "the text is + // there, just outside the extractable surface" — once a widget value has + // been promoted into page content that no longer holds, so the reason must + // not fire and the page must not be routed to OCR to recover text the + // parser already returned. `test_annotation_text_complexity_reason` covers + // the non-widget appearance text the reason still exists for. + let complexity = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + ..Default::default() + }) + .is_complex(PdfInput::Path( + "../../integration_tests_data/filled_acroform.pdf".into(), + )) + .await + .expect("complexity analysis should run on the flattened document"); + assert_eq!(complexity.len(), 3); + assert!( + !complexity[1] + .reasons + .contains(&ComplexityReason::AnnotationText), + "widget text is extractable after flattening, so it is not annotation-only text" + ); +} diff --git a/crates/liteparse/tests/side_by_side_ruled_tables.rs b/crates/liteparse/tests/side_by_side_ruled_tables.rs new file mode 100644 index 00000000..07c5a163 --- /dev/null +++ b/crates/liteparse/tests/side_by_side_ruled_tables.rs @@ -0,0 +1,393 @@ +use liteparse::config::ImageMode; +use liteparse::output::markdown::format_markdown; +use liteparse::projection::project_pages_to_grid; +use liteparse::types::{GraphicPrimitive, Page, TextItem}; + +fn text_item(text: &str, x: f32, y: f32, width: f32, size: f32, bold: bool) -> TextItem { + TextItem { + text: text.to_string(), + x, + y, + width, + height: size * 1.116, + font_name: Some(if bold { + "LiberationSans-Bold".to_string() + } else { + "LiberationSans".to_string() + }), + font_size: Some(size), + font_height: Some(size), + ..TextItem::default() + } +} + +fn stroke(x1: f32, y1: f32, x2: f32, y2: f32) -> GraphicPrimitive { + GraphicPrimitive::Stroke { + x1, + y1, + x2, + y2, + color: Some("ff000000".to_string()), + width: 0.5, + } +} + +fn ruled_grid(xs: &[f32], ys: &[f32]) -> Vec { + let mut graphics = Vec::new(); + for &y in ys { + graphics.push(stroke(xs[0], y, *xs.last().unwrap(), y)); + } + for &x in xs { + graphics.push(stroke(x, ys[0], x, *ys.last().unwrap())); + } + graphics +} + +fn markdown_for(text_items: Vec, graphics: Vec) -> String { + let pages = project_pages_to_grid(vec![Page { + page_number: 1, + page_width: 792.0, + page_height: 612.0, + content_bounds: None, + text_items, + graphics, + vector_graphics: None, + struct_nodes: Vec::new(), + image_refs: Vec::new(), + annotations: None, + form_fields: None, + structure_tree: None, + }]); + format_markdown(&pages, &[], ImageMode::Off) +} + +#[test] +fn side_by_side_ruled_tables_keep_their_own_headings_in_markdown() { + let mut text_items = vec![ + text_item("Public Release Reference", 274.4, 50.4, 243.26, 20.0, true), + text_item( + "A public, synthetic document for evaluating side-by-side table extraction", + 237.45, + 78.7, + 317.13, + 10.0, + false, + ), + text_item( + "This document contains generic release-planning examples only.", + 50.5, + 105.35, + 350.0, + 10.0, + false, + ), + text_item("Release channels", 50.5, 129.99, 100.57, 12.0, true), + text_item("Support windows", 406.9, 130.99, 99.88, 12.0, true), + ]; + + for (y, values) in [ + (154.655, ["Channel", "Status", "Owner"]), + (174.705, ["Stable", "Ready", "Team A"]), + (194.755, ["Beta", "Testing", "Team B"]), + (214.805, ["Nightly", "Active", "Team C"]), + ] { + for ((text, x), width) in values + .into_iter() + .zip([50.5, 118.5, 180.5]) + .zip([35.46, 29.5, 32.0]) + { + text_items.push(text_item(text, x, y, width, 9.0, y == 154.655)); + } + } + + for (y, values) in [ + (155.655, ["Region", "Window", "Contact"]), + (175.705, ["East", "Morning", "Desk 1"]), + (195.755, ["West", "Afternoon", "Desk 2"]), + (215.805, ["Central", "Evening", "Desk 3"]), + ] { + for ((text, x), width) in values + .into_iter() + .zip([406.9, 474.9, 536.9]) + .zip([35.0, 39.0, 34.0]) + { + text_items.push(text_item(text, x, y, width, 9.0, y == 155.655)); + } + } + + let mut graphics = ruled_grid( + &[44.0, 112.0, 174.0, 236.0], + &[148.0, 168.0, 188.0, 208.0, 228.0], + ); + graphics.extend(ruled_grid( + &[400.0, 468.0, 530.0, 594.0], + &[149.0, 169.0, 189.0, 209.0, 229.0], + )); + + let markdown = markdown_for(text_items, graphics); + let expected = "# Public Release Reference\n\n\ +A public, synthetic document for evaluating side-by-side table extraction\n\n\ +This document contains generic release-planning examples only.\n\n\ +## Release channels\n\n\ +| Channel | Status | Owner |\n\ +|---|---|---|\n\ +| Stable | Ready | Team A |\n\ +| Beta | Testing | Team B |\n\ +| Nightly | Active | Team C |\n\n\ +## Support windows\n\n\ +| Region | Window | Contact |\n\ +|---|---|---|\n\ +| East | Morning | Desk 1 |\n\ +| West | Afternoon | Desk 2 |\n\ +| Central | Evening | Desk 3 |"; + + assert_eq!(markdown, expected); +} + +#[test] +fn one_wide_ruled_table_remains_one_table() { + let mut text_items = vec![text_item( + "Quarterly schedule", + 50.0, + 80.0, + 130.0, + 14.0, + true, + )]; + for (y, values) in [ + (124.0, ["Quarter", "Status", "Owner"]), + (148.0, ["Q1", "Ready", "Team A"]), + (172.0, ["Q2", "Testing", "Team B"]), + ] { + for ((text, x), width) in values + .into_iter() + .zip([56.0, 260.0, 464.0]) + .zip([60.0, 70.0, 70.0]) + { + text_items.push(text_item(text, x, y, width, 10.0, y == 124.0)); + } + } + let graphics = ruled_grid(&[50.0, 250.0, 454.0, 660.0], &[116.0, 140.0, 164.0, 188.0]); + + let markdown = markdown_for(text_items, graphics); + let expected = "# Quarterly schedule\n\n\ +---\n\n\ +| Quarter | Status | Owner |\n\ +|---|---|---|\n\ +| Q1 | Ready | Team A |\n\ +| Q2 | Testing | Team B |"; + + assert_eq!(markdown, expected); +} + +#[test] +fn vertically_stacked_ruled_tables_keep_top_to_bottom_order() { + let mut text_items = vec![ + text_item("Operations handbook", 250.0, 35.0, 210.0, 20.0, true), + text_item("Release status", 56.0, 80.0, 100.0, 14.0, true), + text_item("Support status", 56.0, 240.0, 105.0, 14.0, true), + ]; + for (y, values) in [ + (108.0, ["Channel", "Status"]), + (132.0, ["Stable", "Ready"]), + (156.0, ["Beta", "Testing"]), + (268.0, ["Region", "Window"]), + (292.0, ["East", "Morning"]), + (316.0, ["West", "Afternoon"]), + ] { + for ((text, x), width) in values.into_iter().zip([56.0, 206.0]).zip([80.0, 90.0]) { + text_items.push(text_item(text, x, y, width, 10.0, y == 108.0 || y == 268.0)); + } + } + + let mut graphics = ruled_grid(&[50.0, 200.0, 350.0], &[100.0, 124.0, 148.0, 172.0]); + graphics.extend(ruled_grid( + &[50.0, 200.0, 350.0], + &[260.0, 284.0, 308.0, 332.0], + )); + + let markdown = markdown_for(text_items, graphics); + let expected = "# Operations handbook\n\n\ +## Release status\n\n\ +| Channel | Status |\n\ +|---|---|\n\ +| Stable | Ready |\n\ +| Beta | Testing |\n\n\ +---\n\n\ +### Support status\n\n\ +| Region | Window |\n\ +|---|---|\n\ +| East | Morning |\n\ +| West | Afternoon |"; + + assert_eq!(markdown, expected); +} + +#[test] +fn ordinary_two_column_prose_is_not_rendered_as_a_table() { + let mut text_items = vec![text_item( + "Two-column article", + 270.0, + 40.0, + 180.0, + 20.0, + true, + )]; + for row in 0..5 { + let y = 110.0 + row as f32 * 18.0; + text_items.push(text_item( + &format!("Left paragraph sentence number {} continues.", row + 1), + 50.0, + y, + 270.0, + 10.0, + false, + )); + text_items.push(text_item( + &format!("Right paragraph sentence number {} continues.", row + 1), + 430.0, + y + 0.8, + 270.0, + 10.0, + false, + )); + } + + let markdown = markdown_for(text_items, Vec::new()); + + assert!(!markdown.contains("|---"), "unexpected table:\n{markdown}"); +} + +#[test] +fn spanning_heading_above_side_by_side_tables_remains_spanning() { + let mut text_items = vec![text_item( + "Shared service matrix", + 250.0, + 80.0, + 290.0, + 18.0, + true, + )]; + for (y, left, right) in [ + ( + 124.0, + ["Channel", "Status", "Owner"], + ["Region", "Window", "Contact"], + ), + ( + 148.0, + ["Stable", "Ready", "Team A"], + ["East", "Morning", "Desk 1"], + ), + ( + 172.0, + ["Beta", "Testing", "Team B"], + ["West", "Afternoon", "Desk 2"], + ), + ( + 196.0, + ["Nightly", "Active", "Team C"], + ["Central", "Evening", "Desk 3"], + ), + ] { + for ((text, x), width) in left + .into_iter() + .zip([56.0, 136.0, 216.0]) + .zip([60.0, 65.0, 65.0]) + { + text_items.push(text_item(text, x, y, width, 10.0, y == 124.0)); + } + for ((text, x), width) in right + .into_iter() + .zip([426.0, 506.0, 586.0]) + .zip([60.0, 65.0, 65.0]) + { + text_items.push(text_item(text, x, y + 0.8, width, 10.0, y == 124.0)); + } + } + + let mut graphics = ruled_grid( + &[50.0, 130.0, 210.0, 290.0], + &[116.0, 140.0, 164.0, 188.0, 212.0], + ); + graphics.extend(ruled_grid( + &[420.0, 500.0, 580.0, 660.0], + &[116.8, 140.8, 164.8, 188.8, 212.8], + )); + + let markdown = markdown_for(text_items, graphics); + let expected = "# Shared service matrix\n\n\ +---\n\n\ +| Channel | Status | Owner |\n\ +|---|---|---|\n\ +| Stable | Ready | Team A |\n\ +| Beta | Testing | Team B |\n\ +| Nightly | Active | Team C |\n\n\ +| Region | Window | Contact |\n\ +|---|---|---|\n\ +| East | Morning | Desk 1 |\n\ +| West | Afternoon | Desk 2 |\n\ +| Central | Evening | Desk 3 |"; + + assert_eq!(markdown, expected); +} + +#[test] +fn spanning_line_between_rows_does_not_split_side_by_side_tables() { + // A page-spanning note sitting vertically between the data rows of two + // side-by-side grids belongs to neither one, so it has no grid rank to + // sort by. Reordering the tables around it must not strand it mid-table: + // every row of both grids has to survive as table content. + let mut text_items = vec![text_item( + "Shared service matrix", + 250.0, + 80.0, + 290.0, + 18.0, + true, + )]; + for row in 0..6 { + let y = 124.0 + row as f32 * 24.0; + for ((text, x), width) in [format!("L{row}a"), format!("L{row}b"), format!("L{row}c")] + .into_iter() + .zip([56.0, 136.0, 216.0]) + .zip([60.0, 65.0, 65.0]) + { + text_items.push(text_item(&text, x, y, width, 10.0, row == 0)); + } + for ((text, x), width) in [format!("R{row}a"), format!("R{row}b"), format!("R{row}c")] + .into_iter() + .zip([426.0, 506.0, 586.0]) + .zip([60.0, 65.0, 65.0]) + { + text_items.push(text_item(&text, x, y + 0.8, width, 10.0, row == 0)); + } + } + text_items.push(text_item( + "Note: spanning remark placed after the second data row of both tables.", + 56.0, + 184.0, + 604.0, + 10.0, + false, + )); + + let ys = [116.0, 140.0, 164.0, 188.0, 212.0, 236.0, 260.0]; + let mut graphics = ruled_grid(&[50.0, 130.0, 210.0, 290.0], &ys); + graphics.extend(ruled_grid( + &[420.0, 500.0, 580.0, 660.0], + &ys.map(|y| y + 0.8), + )); + + let markdown = markdown_for(text_items, graphics); + for row in 0..6 { + for label in [format!("L{row}a"), format!("R{row}a")] { + assert!( + markdown + .lines() + .any(|line| line.starts_with('|') && line.contains(&label)), + "{label} is not table content:\n{markdown}" + ); + } + } +} diff --git a/crates/pdfium-sys/Cargo.toml b/crates/pdfium-sys/Cargo.toml index 81893fee..b9c41ed8 100644 --- a/crates/pdfium-sys/Cargo.toml +++ b/crates/pdfium-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "liteparse-pdfium-sys" -version = "1.4.0" +version = "1.7.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/crates/pdfium-sys/bindings.rs b/crates/pdfium-sys/bindings.rs index 27a04774..e6f46c17 100644 --- a/crates/pdfium-sys/bindings.rs +++ b/crates/pdfium-sys/bindings.rs @@ -99,6 +99,11 @@ pub const FPDF_FORMFIELD_LISTBOX: u32 = 5; pub const FPDF_FORMFIELD_TEXTFIELD: u32 = 6; pub const FPDF_FORMFIELD_SIGNATURE: u32 = 7; pub const FPDF_FORMFIELD_COUNT: u32 = 8; +pub const FLATTEN_FAIL: u32 = 0; +pub const FLATTEN_SUCCESS: u32 = 1; +pub const FLATTEN_NOTHINGTODO: u32 = 2; +pub const FLAT_NORMALDISPLAY: u32 = 0; +pub const FLAT_PRINT: u32 = 1; pub const FPDF_ANNOT_UNKNOWN: u32 = 0; pub const FPDF_ANNOT_TEXT: u32 = 1; pub const FPDF_ANNOT_LINK: u32 = 2; @@ -563,6 +568,22 @@ unsafe extern "C" { fileVersion: *mut ::std::os::raw::c_int, ) -> FPDF_BOOL; } +unsafe extern "C" { + pub fn FPDF_GetSignatureCount(document: FPDF_DOCUMENT) -> ::std::os::raw::c_int; +} +unsafe extern "C" { + pub fn FPDF_GetSignatureObject( + document: FPDF_DOCUMENT, + index: ::std::os::raw::c_int, + ) -> FPDF_SIGNATURE; +} +unsafe extern "C" { + pub fn FPDFSignatureObj_GetByteRange( + signature: FPDF_SIGNATURE, + buffer: *mut ::std::os::raw::c_int, + length: ::std::os::raw::c_ulong, + ) -> ::std::os::raw::c_ulong; +} unsafe extern "C" { pub fn FPDF_GetLastError() -> ::std::os::raw::c_ulong; } @@ -1039,6 +1060,12 @@ pub struct FPDF_IMAGEOBJ_METADATA { pub colorspace: ::std::os::raw::c_int, pub marked_content_id: ::std::os::raw::c_int, } +unsafe extern "C" { + pub fn FPDFPage_Flatten( + page: FPDF_PAGE, + nFlag: ::std::os::raw::c_int, + ) -> ::std::os::raw::c_int; +} unsafe extern "C" { pub fn FPDF_CreateNewDocument() -> FPDF_DOCUMENT; } diff --git a/crates/pdfium-sys/build.rs b/crates/pdfium-sys/build.rs index 7cfdbb4c..dc507b47 100644 --- a/crates/pdfium-sys/build.rs +++ b/crates/pdfium-sys/build.rs @@ -277,6 +277,7 @@ fn run_bindgen(include_dir: &Path) { .allowlist_type("FPDF.*") .allowlist_type("FS_.*") .allowlist_var("FPDF.*") + .allowlist_var("FLAT.*") .derive_debug(true) .derive_default(true) .layout_tests(false) diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rs index be24f030..d0ae4763 100644 --- a/crates/pdfium-sys/src/dynamic.rs +++ b/crates/pdfium-sys/src/dynamic.rs @@ -23,6 +23,17 @@ macro_rules! load_fn { }}; } +/// Like `load_fn!`, but yields `None` instead of failing the whole load when +/// the symbol is missing. For APIs a trimmed pdfium build may omit — callers +/// must degrade gracefully rather than assume the function exists. +macro_rules! load_fn_opt { + ($lib:expr, $name:literal) => {{ + unsafe { $lib.get::<*const ()>($name.as_bytes()) } + .ok() + .map(|sym| unsafe { std::mem::transmute(*sym) }) + }}; +} + /// Holds all pdfium function pointers loaded at runtime. pub struct PdfiumBindings { // Keep the library handle alive — dropping it would unload the symbols. @@ -67,6 +78,22 @@ pub struct PdfiumBindings { *mut std::os::raw::c_void, std::os::raw::c_ulong, ) -> std::os::raw::c_ulong, + pub FPDF_GetFileVersion: + unsafe extern "C" fn(FPDF_DOCUMENT, *mut std::os::raw::c_int) -> FPDF_BOOL, + pub FPDF_GetSecurityHandlerRevision: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_int, + pub FPDF_GetDocPermissions: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_ulong, + // `fpdf_signature` is absent from some trimmed pdfium builds, so these + // three load optionally and callers fall back to "no signature info". + pub FPDF_GetSignatureCount: Option std::os::raw::c_int>, + pub FPDF_GetSignatureObject: + Option FPDF_SIGNATURE>, + pub FPDFSignatureObj_GetByteRange: Option< + unsafe extern "C" fn( + FPDF_SIGNATURE, + *mut std::os::raw::c_int, + std::os::raw::c_ulong, + ) -> std::os::raw::c_ulong, + >, pub FPDF_GetXFAPacketCount: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_int, pub FPDF_GetXFAPacketName: unsafe extern "C" fn( FPDF_DOCUMENT, @@ -89,6 +116,8 @@ pub struct PdfiumBindings { pub FPDF_GetPageHeightF: unsafe extern "C" fn(FPDF_PAGE) -> f32, pub FPDF_GetPageBoundingBox: unsafe extern "C" fn(FPDF_PAGE, *mut FS_RECTF) -> FPDF_BOOL, pub FPDFPage_GetRotation: unsafe extern "C" fn(FPDF_PAGE) -> std::os::raw::c_int, + pub FPDFPage_Flatten: + Option std::os::raw::c_int>, pub FPDF_PageToDevice: unsafe extern "C" fn( FPDF_PAGE, std::os::raw::c_int, @@ -367,6 +396,8 @@ pub struct PdfiumBindings { unsafe extern "C" fn(FPDF_ANNOTATION, FPDF_BYTESTRING) -> FPDF_ANNOTATION, pub FPDFAnnot_GetObjNum: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int, pub FPDFAnnot_GetFlags: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int, + pub FPDFAnnot_SetFlags: + Option FPDF_BOOL>, pub FPDFAnnot_GetObjectCount: unsafe extern "C" fn(FPDF_ANNOTATION) -> std::os::raw::c_int, pub FPDFAnnot_GetObject: unsafe extern "C" fn(FPDF_ANNOTATION, std::os::raw::c_int) -> FPDF_PAGEOBJECT, @@ -528,6 +559,12 @@ impl PdfiumBindings { FORM_DoPageAAction: load_fn!(lib, "FORM_DoPageAAction"), FPDF_FFLDraw: load_fn!(lib, "FPDF_FFLDraw"), FPDF_GetMetaText: load_fn!(lib, "FPDF_GetMetaText"), + FPDF_GetFileVersion: load_fn!(lib, "FPDF_GetFileVersion"), + FPDF_GetSecurityHandlerRevision: load_fn!(lib, "FPDF_GetSecurityHandlerRevision"), + FPDF_GetDocPermissions: load_fn!(lib, "FPDF_GetDocPermissions"), + FPDF_GetSignatureCount: load_fn_opt!(lib, "FPDF_GetSignatureCount"), + FPDF_GetSignatureObject: load_fn_opt!(lib, "FPDF_GetSignatureObject"), + FPDFSignatureObj_GetByteRange: load_fn_opt!(lib, "FPDFSignatureObj_GetByteRange"), FPDF_GetXFAPacketCount: load_fn!(lib, "FPDF_GetXFAPacketCount"), FPDF_GetXFAPacketName: load_fn!(lib, "FPDF_GetXFAPacketName"), FPDF_GetXFAPacketContent: load_fn!(lib, "FPDF_GetXFAPacketContent"), @@ -537,6 +574,7 @@ impl PdfiumBindings { FPDF_GetPageHeightF: load_fn!(lib, "FPDF_GetPageHeightF"), FPDF_GetPageBoundingBox: load_fn!(lib, "FPDF_GetPageBoundingBox"), FPDFPage_GetRotation: load_fn!(lib, "FPDFPage_GetRotation"), + FPDFPage_Flatten: load_fn_opt!(lib, "FPDFPage_Flatten"), FPDF_PageToDevice: load_fn!(lib, "FPDF_PageToDevice"), FPDFPage_CountObjects: load_fn!(lib, "FPDFPage_CountObjects"), FPDFPage_GetObject: load_fn!(lib, "FPDFPage_GetObject"), @@ -634,6 +672,7 @@ impl PdfiumBindings { FPDFAnnot_GetLinkedAnnot: load_fn!(lib, "FPDFAnnot_GetLinkedAnnot"), FPDFAnnot_GetObjNum: load_fn!(lib, "FPDFAnnot_GetObjNum"), FPDFAnnot_GetFlags: load_fn!(lib, "FPDFAnnot_GetFlags"), + FPDFAnnot_SetFlags: load_fn_opt!(lib, "FPDFAnnot_SetFlags"), FPDFAnnot_GetObjectCount: load_fn!(lib, "FPDFAnnot_GetObjectCount"), FPDFAnnot_GetObject: load_fn!(lib, "FPDFAnnot_GetObject"), FPDFAnnot_GetFormFieldFlags: load_fn!(lib, "FPDFAnnot_GetFormFieldFlags"), diff --git a/crates/pdfium-sys/wrapper.h b/crates/pdfium-sys/wrapper.h index e35d8416..10cbd3f6 100644 --- a/crates/pdfium-sys/wrapper.h +++ b/crates/pdfium-sys/wrapper.h @@ -5,3 +5,5 @@ #include "fpdf_annot.h" #include "fpdf_structtree.h" #include "fpdf_transformpage.h" +#include "fpdf_signature.h" +#include "fpdf_flatten.h" diff --git a/crates/pdfium/Cargo.toml b/crates/pdfium/Cargo.toml index 6b3192a9..a198f242 100644 --- a/crates/pdfium/Cargo.toml +++ b/crates/pdfium/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "liteparse-pdfium" -version = "1.4.0" +version = "1.7.0" edition.workspace = true license.workspace = true repository.workspace = true description = "Safe Rust wrapper around PDFium for liteparse" [dependencies] -pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.4.0", path = "../pdfium-sys" } +pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.7.0", path = "../pdfium-sys" } [dev-dependencies] blake3 = "1" diff --git a/crates/pdfium/src/document.rs b/crates/pdfium/src/document.rs index 7e7f18ef..80d9bf46 100644 --- a/crates/pdfium/src/document.rs +++ b/crates/pdfium/src/document.rs @@ -50,6 +50,52 @@ pub struct XfaPacket { pub content: Option>, } +/// Signature summary used for document provenance metadata. +#[derive(Debug, Clone, Copy, Default)] +pub struct SignatureSummary { + /// `None` when the loaded pdfium build has no signature API, which is not + /// the same as a document with zero signatures. + pub count: Option, + /// `None` when signatures exist but PDFium did not expose any byte range. + pub byte_range_reaches_eof: Option, +} + +/// The `fpdf_signature` entry points, resolved together. `None` when the +/// loaded pdfium build does not export them. +struct SignatureApi { + count: unsafe extern "C" fn(pdfium_sys::FPDF_DOCUMENT) -> std::os::raw::c_int, + object: unsafe extern "C" fn( + pdfium_sys::FPDF_DOCUMENT, + std::os::raw::c_int, + ) -> pdfium_sys::FPDF_SIGNATURE, + byte_range: unsafe extern "C" fn( + pdfium_sys::FPDF_SIGNATURE, + *mut std::os::raw::c_int, + std::os::raw::c_ulong, + ) -> std::os::raw::c_ulong, +} + +impl SignatureApi { + #[cfg(not(target_arch = "wasm32"))] + fn load() -> Option { + let bindings = pdfium_sys::dynamic::pdfium(); + Some(Self { + count: bindings.FPDF_GetSignatureCount?, + object: bindings.FPDF_GetSignatureObject?, + byte_range: bindings.FPDFSignatureObj_GetByteRange?, + }) + } + + #[cfg(target_arch = "wasm32")] + fn load() -> Option { + Some(Self { + count: pdfium_sys::FPDF_GetSignatureCount, + object: pdfium_sys::FPDF_GetSignatureObject, + byte_range: pdfium_sys::FPDFSignatureObj_GetByteRange, + }) + } +} + impl<'lib> Document<'lib> { pub fn page_count(&self) -> i32 { unsafe { ffi!(FPDF_GetPageCount(self.handle)) } @@ -92,6 +138,24 @@ impl<'lib> Document<'lib> { }) } + /// Flatten the visible form-widget appearances on `index` into the page + /// content stream and hand back a freshly loaded page reflecting them. + /// + /// Flattening mutates this document in place and invalidates the page + /// handle it ran on, so the load/flatten/reload sequence lives here rather + /// than at call sites where a stale handle would be easy to keep using. + /// Returns `Ok(None)` when nothing was flattened — the caller should keep + /// using its existing page. + pub fn flatten_form_widgets(&self, index: i32) -> Result>, PdfiumError> { + { + let page = self.page(index)?; + if !page.flatten_form_widgets_for_display() { + return Ok(None); + } + } + self.page(index).map(Some) + } + /// Read one entry from the document's `/Info` metadata dictionary /// (e.g. `"Creator"`, `"Producer"`, `"Title"`). Returns `None` when the /// tag is absent or empty. @@ -133,6 +197,76 @@ impl<'lib> Document<'lib> { Some(String::from_utf16_lossy(&buf[..end])) } + /// Encoded PDF version (`14` means PDF 1.4), when present. + pub fn file_version(&self) -> Option { + let mut version = 0; + let ok = unsafe { ffi!(FPDF_GetFileVersion(self.handle, &mut version)) }; + (ok != 0).then_some(version) + } + + /// PDF security-handler revision, or `-1` for an unencrypted document. + pub fn security_handler_revision(&self) -> i32 { + unsafe { ffi!(FPDF_GetSecurityHandlerRevision(self.handle)) } + } + + /// Document permission flags reported by PDFium. + pub fn permissions(&self) -> u64 { + unsafe { ffi!(FPDF_GetDocPermissions(self.handle)) as u64 } + } + + /// Count signatures and determine whether every readable final byte-range + /// segment reaches the current end of the file. Needs `file_size` to answer + /// the byte-range question at all — without it the verdict stays `None` + /// rather than defaulting to "reaches EOF". + pub fn signature_summary(&self, file_size: Option) -> SignatureSummary { + const MAX_BYTE_RANGE_VALUES: usize = 8; + let Some(api) = SignatureApi::load() else { + return SignatureSummary::default(); + }; + let count = unsafe { (api.count)(self.handle) }.max(0) as u32; + let Some(file_size) = file_size.filter(|_| count > 0) else { + return SignatureSummary { + count: Some(count), + byte_range_reaches_eof: None, + }; + }; + + let mut known = false; + let mut reaches_eof = true; + for index in 0..count { + let signature = unsafe { (api.object)(self.handle, index as i32) }; + if signature.is_null() { + continue; + } + let mut ranges = [0i32; MAX_BYTE_RANGE_VALUES]; + let len = unsafe { + (api.byte_range)( + signature, + ranges.as_mut_ptr(), + ranges.len() as std::os::raw::c_ulong, + ) + } as usize; + if !(2..=MAX_BYTE_RANGE_VALUES).contains(&len) { + continue; + } + known = true; + let start = i64::from(ranges[len - 2]); + let length = i64::from(ranges[len - 1]); + if start < 0 + || length < 0 + || u64::try_from(start + length) + .ok() + .is_some_and(|range_end| range_end < file_size) + { + reaches_eof = false; + } + } + SignatureSummary { + count: Some(count), + byte_range_reaches_eof: known.then_some(reaches_eof), + } + } + /// Number of packets in the document's `/XFA` array (0 for non-XFA docs). pub fn xfa_packet_count(&self) -> i32 { unsafe { ffi!(FPDF_GetXFAPacketCount(self.handle)) } diff --git a/crates/pdfium/src/lib.rs b/crates/pdfium/src/lib.rs index 48f607da..3ce12e32 100644 --- a/crates/pdfium/src/lib.rs +++ b/crates/pdfium/src/lib.rs @@ -9,7 +9,7 @@ mod text_page; mod types; pub use bitmap::Bitmap; -pub use document::{Document, FormEnvironment, OutlineEntry, XfaPacket}; +pub use document::{Document, FormEnvironment, OutlineEntry, SignatureSummary, XfaPacket}; pub use error::PdfiumError; pub use font::{Font, FontType}; pub use library::Library; diff --git a/crates/pdfium/src/page.rs b/crates/pdfium/src/page.rs index a967d391..4f0fdad0 100644 --- a/crates/pdfium/src/page.rs +++ b/crates/pdfium/src/page.rs @@ -243,6 +243,17 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> { unsafe { ffi!(FPDFPage_GetRotation(self.handle)) } } + /// Page dimensions in the same rotation-adjusted viewport coordinate + /// space returned by [`Self::page_to_viewport`]. + pub fn viewport_size(&self, view_box: &RectF) -> (f32, f32) { + let mut width = (view_box.right - view_box.left).abs(); + let mut height = (view_box.top - view_box.bottom).abs(); + if matches!(self.rotation(), 1 | 3) { + std::mem::swap(&mut width, &mut height); + } + (width, height) + } + /// Get the page bounding box (CropBox, falls back to MediaBox). /// Coordinates in PDF page space. pub fn view_box(&self) -> Option { @@ -268,14 +279,7 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> { /// Convert a point from PDF page space to viewport space (top-left origin, 72 DPI). /// Mirrors the platform's Parse_pageToViewport using FPDF_PageToDevice at 1000x scale. pub fn page_to_viewport(&self, view_box: &RectF, page_x: f32, page_y: f32) -> (f32, f32) { - let mut vw = view_box.right - view_box.left; - let mut vh = view_box.top - view_box.bottom; - - let rotation = self.rotation(); - if rotation == 1 || rotation == 3 { - // 90° or 270° — swap viewport dimensions - std::mem::swap(&mut vw, &mut vh); - } + let (vw, vh) = self.viewport_size(view_box); let device_w = (vw * 1000.0).round() as i32; let device_h = (vh * 1000.0).round() as i32; @@ -911,30 +915,160 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> { let subtype = unsafe { ffi!(FPDFAnnot_GetSubtype(annot)) }; let flags = unsafe { ffi!(FPDFAnnot_GetFlags(annot)) }; let hidden = flags & pdfium_sys::FPDF_ANNOT_FLAG_HIDDEN as i32 != 0; - let mut found = false; - if !hidden && subtype != pdfium_sys::FPDF_ANNOT_POPUP as i32 { - let object_count = unsafe { ffi!(FPDFAnnot_GetObjectCount(annot)) }; - for object_index in 0..object_count { - let object = unsafe { ffi!(FPDFAnnot_GetObject(annot, object_index)) }; - if object.is_null() { - continue; - } - if unsafe { ffi!(FPDFPageObj_GetType(object)) } - == pdfium_sys::FPDF_PAGEOBJ_TEXT as i32 - { - found = true; - break; - } + let found = !hidden + && subtype != pdfium_sys::FPDF_ANNOT_POPUP as i32 + && annotation_paints_text_shallow(annot); + unsafe { ffi!(FPDFPage_CloseAnnot(annot)) }; + if found { + return true; + } + } + false + } + + /// Viewport rects of the visible AcroForm widgets that paint text through + /// their appearance streams. Empty when the page has no such widget, which + /// is the signal not to flatten. + /// + /// PDFium's page text API omits these glyphs until the page is flattened, + /// so the rects double as the only regions where flattening can introduce + /// text — callers use them to scope duplicate detection instead of + /// rescanning the whole page. + pub fn form_widget_text_rects(&self, view_box: &RectF) -> Vec { + let mut rects = Vec::new(); + let count = unsafe { ffi!(FPDFPage_GetAnnotCount(self.handle)) }; + for index in 0..count { + let annot = unsafe { ffi!(FPDFPage_GetAnnot(self.handle, index)) }; + if annot.is_null() { + continue; + } + if unsafe { ffi!(FPDFAnnot_GetSubtype(annot)) } == pdfium_sys::FPDF_ANNOT_WIDGET as i32 + && annotation_paints_text_deep(annot) + { + let mut rect = pdfium_sys::FS_RECTF::default(); + if unsafe { ffi!(FPDFAnnot_GetRect(annot, &mut rect)) } != 0 { + rects.push(self.bounds_to_viewport( + view_box, + &RectF { + left: rect.left, + top: rect.top, + right: rect.right, + bottom: rect.bottom, + }, + )); } } unsafe { ffi!(FPDFPage_CloseAnnot(annot)) }; - if found { + } + rects + } + + /// Whether any text object already in the page content stream overlaps one + /// of `rects`. + /// + /// Flattening replaces the page content under a widget's rect with that + /// widget's appearance, so text already drawn there is lost. This is the + /// cheap probe for that situation: it walks page-object bounding boxes + /// only — no text page, no glyph decoding — so the common form page (whose + /// widget rects sit over blank space) pays a bounds walk instead of a + /// second full text extraction. + pub fn text_objects_overlap(&self, view_box: &RectF, rects: &[RectF]) -> bool { + if rects.is_empty() { + return false; + } + let count = unsafe { ffi!(FPDFPage_CountObjects(self.handle)) }; + for i in 0..count { + let obj = unsafe { ffi!(FPDFPage_GetObject(self.handle, i)) }; + if obj.is_null() + || unsafe { ffi!(FPDFPageObj_GetType(obj)) } != pdfium_sys::FPDF_PAGEOBJ_TEXT as i32 + { + continue; + } + let (mut left, mut bottom, mut right, mut top) = (0.0f32, 0.0f32, 0.0f32, 0.0f32); + if unsafe { + ffi!(FPDFPageObj_GetBounds( + obj, + &mut left, + &mut bottom, + &mut right, + &mut top + )) + } == 0 + { + continue; + } + let bounds = self.bounds_to_viewport( + view_box, + &RectF { + left, + top, + right, + bottom, + }, + ); + if rects.iter().any(|rect| { + bounds.left < rect.right + && bounds.right > rect.left + && bounds.top < rect.bottom + && bounds.bottom > rect.top + }) { return true; } } false } + /// Promote visible form-widget appearances into page content without + /// admitting comment, markup, stamp, or other annotation appearances into + /// the text layer. + /// + /// PDFium's flatten operation is page-wide and otherwise consumes every + /// visible annotation. Hide non-widget annotations in this disposable + /// extraction document first; callers snapshot annotation metadata and + /// reopen the pristine input for any later rendering work. + /// + /// Returns true only when PDFium changed the page. Returns false — leaving + /// extraction on the original page content — when the pdfium build omits + /// the flatten API, or when suppression or flattening fails, in which case + /// any changed flags are restored first. + pub fn flatten_form_widgets_for_display(&self) -> bool { + // `fpdf_flatten.h` is an optional pdfium API; trimmed builds omit it. + // Missing it costs form-value text, not the whole parse. + let Some(api) = FlattenApi::load() else { + return false; + }; + let mut suppressed = Vec::new(); + let count = unsafe { ffi!(FPDFPage_GetAnnotCount(self.handle)) }; + for index in 0..count { + let annot = unsafe { ffi!(FPDFPage_GetAnnot(self.handle, index)) }; + if annot.is_null() { + continue; + } + let subtype = unsafe { ffi!(FPDFAnnot_GetSubtype(annot)) }; + if subtype != pdfium_sys::FPDF_ANNOT_WIDGET as i32 { + let flags = unsafe { ffi!(FPDFAnnot_GetFlags(annot)) }; + if flags & pdfium_sys::FPDF_ANNOT_FLAG_HIDDEN as i32 == 0 { + let hidden_flags = flags | pdfium_sys::FPDF_ANNOT_FLAG_HIDDEN as i32; + let changed = unsafe { (api.set_flags)(annot, hidden_flags) } != 0; + unsafe { ffi!(FPDFPage_CloseAnnot(annot)) }; + if !changed { + restore_annotation_flags(&api, self.handle, &suppressed); + return false; + } + suppressed.push((index, flags)); + continue; + } + } + unsafe { ffi!(FPDFPage_CloseAnnot(annot)) }; + } + + let result = unsafe { (api.flatten)(self.handle, pdfium_sys::FLAT_NORMALDISPLAY as i32) }; + if result != pdfium_sys::FLATTEN_SUCCESS as i32 { + restore_annotation_flags(&api, self.handle, &suppressed); + } + result == pdfium_sys::FLATTEN_SUCCESS as i32 + } + /// Enumerate AcroForm widget annotations and resolve their field values /// through PDFium's form-fill environment. pub fn form_fields( @@ -1087,6 +1221,115 @@ impl<'doc, 'lib: 'doc> Page<'doc, 'lib> { } } +/// The optional page-flatten API, resolved together so a build missing either +/// half degrades to "no flattening" rather than failing the whole pdfium load. +struct FlattenApi { + flatten: + unsafe extern "C" fn(pdfium_sys::FPDF_PAGE, std::os::raw::c_int) -> std::os::raw::c_int, + set_flags: unsafe extern "C" fn( + pdfium_sys::FPDF_ANNOTATION, + std::os::raw::c_int, + ) -> pdfium_sys::FPDF_BOOL, +} + +impl FlattenApi { + #[cfg(not(target_arch = "wasm32"))] + fn load() -> Option { + let bindings = pdfium_sys::dynamic::pdfium(); + Some(Self { + flatten: bindings.FPDFPage_Flatten?, + set_flags: bindings.FPDFAnnot_SetFlags?, + }) + } + + #[cfg(target_arch = "wasm32")] + fn load() -> Option { + Some(Self { + flatten: pdfium_sys::FPDFPage_Flatten, + set_flags: pdfium_sys::FPDFAnnot_SetFlags, + }) + } +} + +/// Whether the annotation's appearance paints text at its top level. +/// +/// Deliberately shallow and HIDDEN-agnostic: this backs the long-standing +/// `AnnotationText` complexity signal, and widening it would silently reroute +/// pages to OCR. [`annotation_paints_text_deep`] is the form-widget variant. +fn annotation_paints_text_shallow(annot: pdfium_sys::FPDF_ANNOTATION) -> bool { + let object_count = unsafe { ffi!(FPDFAnnot_GetObjectCount(annot)) }; + (0..object_count).any(|object_index| { + let object = unsafe { ffi!(FPDFAnnot_GetObject(annot, object_index)) }; + !object.is_null() + && unsafe { ffi!(FPDFPageObj_GetType(object)) } == pdfium_sys::FPDF_PAGEOBJ_TEXT as i32 + }) +} + +/// Whether a widget's appearance paints text, descending into nested form +/// XObjects. +/// +/// PDFium parses an `/AP /N` stream into top-level objects, so a producer that +/// wraps variable text in `/Tx BMC ... /Fm0 Do EMC` (Acrobat and several +/// server-side fillers do) yields a form object, not a text object. Without the +/// descent those filled fields look empty and never get flattened. +fn annotation_paints_text_deep(annot: pdfium_sys::FPDF_ANNOTATION) -> bool { + // Invisible/hidden/noview widgets are not painted, so flattening them would + // introduce text the reader never sees. + let flags = unsafe { ffi!(FPDFAnnot_GetFlags(annot)) }; + let suppressed = pdfium_sys::FPDF_ANNOT_FLAG_INVISIBLE + | pdfium_sys::FPDF_ANNOT_FLAG_HIDDEN + | pdfium_sys::FPDF_ANNOT_FLAG_NOVIEW; + if flags & suppressed as i32 != 0 { + return false; + } + + let object_count = unsafe { ffi!(FPDFAnnot_GetObjectCount(annot)) }; + (0..object_count).any(|object_index| { + let object = unsafe { ffi!(FPDFAnnot_GetObject(annot, object_index)) }; + !object.is_null() && object_paints_text(object, 0) + }) +} + +/// Depth-bounded search for a text object, following form XObjects. +fn object_paints_text(object: pdfium_sys::FPDF_PAGEOBJECT, depth: u32) -> bool { + // Appearance nesting is shallow in practice; the cap only guards against + // pathological or cyclic documents. + const MAX_DEPTH: u32 = 8; + match unsafe { ffi!(FPDFPageObj_GetType(object)) } as u32 { + pdfium_sys::FPDF_PAGEOBJ_TEXT => true, + pdfium_sys::FPDF_PAGEOBJ_FORM if depth < MAX_DEPTH => { + let count = unsafe { ffi!(FPDFFormObj_CountObjects(object)) }; + (0..count).any(|index| { + let child = unsafe { + ffi!(FPDFFormObj_GetObject( + object, + index as std::os::raw::c_ulong + )) + }; + !child.is_null() && object_paints_text(child, depth + 1) + }) + } + _ => false, + } +} + +fn restore_annotation_flags( + api: &FlattenApi, + page: pdfium_sys::FPDF_PAGE, + originals: &[(i32, i32)], +) { + for &(index, flags) in originals { + let annot = unsafe { ffi!(FPDFPage_GetAnnot(page, index)) }; + if annot.is_null() { + continue; + } + unsafe { + (api.set_flags)(annot, flags); + ffi!(FPDFPage_CloseAnnot(annot)); + } + } +} + fn form_field_type_name(field_type: i32) -> &'static str { match field_type as u32 { pdfium_sys::FPDF_FORMFIELD_PUSHBUTTON => "pushbutton", diff --git a/docs/src/content/docs/liteparse/guides/agent-skill.md b/docs/src/content/docs/liteparse/guides/agent-skill.md index 7a799095..e9edb0a1 100644 --- a/docs/src/content/docs/liteparse/guides/agent-skill.md +++ b/docs/src/content/docs/liteparse/guides/agent-skill.md @@ -5,19 +5,50 @@ sidebar: order: 11 --- -LiteParse can be installed as a **coding agent skill** using Vercel's [skills](https://github.com/vercel-labs/skills) utility. This gives your coding agent the ability to process documents, generate screenshots, and parse text from files, all locally. +LiteParse ships as a **coding agent skill** — a self-contained capability that Claude Code, Cursor, +Codex, and other compatible agents load on demand. Installing it gives your agent the ability to +parse documents, extract text and bounding boxes, and generate screenshots, all locally and with no +API key. ## Installation -Add the LiteParse skill to your project: +Add the LiteParse skill to your project with the [`skills`](https://github.com/vercel-labs/skills) +CLI: ```bash npx skills add run-llama/llamaparse-agent-skills --skill liteparse ``` -This downloads a skill file that compatible coding agents (Claude Code, Cursor, etc.) will automatically pick up. +This downloads a skill file that compatible coding agents pick up automatically. -Once configured, your agent will be able to call the LiteParse CLI commands directly from its code execution environment. This means you can have your agent parse PDFs, pull out the text, and generate screenshots on the fly as part of its reasoning process. +You can also copy [`SKILL.md`](https://github.com/run-llama/llamaparse-agent-skills/blob/main/skills/liteparse/SKILL.md) +into your own skills setup by hand, or install the `liteparse` plugin from the +[agent plugins marketplace](https://github.com/run-llama/llamaparse-agent-plugins) to enable it from +inside Claude Code or Codex. + +### Requirements + +| Requirement | Needed for | +| --- | --- | +| Node 18+ and `npm i -g @llamaindex/liteparse` | The `lit` CLI the skill drives — verify with `lit --version` | +| [LibreOffice](https://www.libreoffice.org/) | Parsing Office files (DOCX, XLSX, PPTX) | +| [`uv`](https://docs.astral.sh/uv/) | The bundled ranked-search helper | + +No API key is required since everything runs on your machine. + +## What the skill teaches + +The skill is more than an install: it encodes the extraction patterns that keep an agent's context +small and its runs cheap. Without it, agents commonly re-parse the same document on every search and +dump entire pages into the conversation. + +- **Parse once to a file, then search that file.** Each `lit parse` re-extracts the whole document, + so the skill has the agent parse to a temp file and run all subsequent searches against it. +- **Minimize round-trips.** Fetch a match and its surrounding context in a single command, and batch + independent lookups together rather than spending a turn per search term. +- **Bound every output.** Results are capped so a single lookup can't flood the context window. +- **Escalate to ranked search** when keyword matching stalls, instead of firing off keyword variants + one turn at a time. ## Example prompts @@ -28,3 +59,9 @@ Once the skill is installed, you can ask your coding agent things like: - "Screenshot pages 1-5 of this PDF at 300 DPI" - "Parse this scanned document using the PaddleOCR server on localhost:8828" - "Get the bounding boxes for all text on page 3" + +## Related + +LiteParse is one of several ways to give an agent document-processing capabilities. For cloud parsing +of complex documents, MCP servers, and workflow nodes, see +[Using LlamaIndex with AI Agents](/for-agents/). diff --git a/docs/src/content/docs/liteparse/index.md b/docs/src/content/docs/liteparse/index.md index 4047bfa0..59220b02 100644 --- a/docs/src/content/docs/liteparse/index.md +++ b/docs/src/content/docs/liteparse/index.md @@ -31,5 +31,6 @@ LiteParse is designed specifically for use cases that require fast, accurate tex - [Document complexity](/liteparse/guides/complexity/): Detect scanned, multi-column, and table-heavy pages up front. - [Library usage](/liteparse/guides/library-usage/): Use LiteParse from TypeScript or Python code. - [Browser usage (WASM)](/liteparse/guides/browser-usage/): Run LiteParse in the browser with zero server dependencies. +- [Agent skill](/liteparse/guides/agent-skill/): Give Claude Code, Cursor, or Codex the ability to parse documents locally. - [CLI reference](/liteparse/cli-reference/): Complete command and option reference. - [API reference](/liteparse/api/): Detailed API documentation (rust) for all public types and functions. The same types apply across all language bindings. diff --git a/integration_tests_data/filled_acroform.pdf b/integration_tests_data/filled_acroform.pdf new file mode 100644 index 00000000..11892f34 Binary files /dev/null and b/integration_tests_data/filled_acroform.pdf differ diff --git a/packages/node/README.md b/packages/node/README.md index 12aeeb8b..52c9b8cb 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -18,6 +18,7 @@ import { LiteParse } from '@llamaindex/liteparse'; const parser = new LiteParse(); const result = await parser.parse('document.pdf'); console.log(result.text); +console.log(`Source document pages: ${result.totalPages}`); // Access structured data for (const page of result.pages) { @@ -25,6 +26,27 @@ for (const page of result.pages) { } ``` +### Bounded-memory parsing + +For documents with many text items, consume page batches without retaining +earlier results: + +```typescript +const parser = new LiteParse(); +for await (const batch of parser.parseBatches('large.pdf', { batchSize: 20 })) { + await processPages(batch.result.pages); +} +``` + +Each batch is an ordinary parse result covering `batch.startPage` through +`batch.endPage`, and becomes collectible as soon as you advance the iterator. +A non-PDF source is converted once, not once per batch. + +Cross-page passes only see the pages in their own batch, so repeated +header/footer removal and image deduplication are batch-local and the output +can differ from `parse()`. Prefer `parse()` unless the size of the +materialized result is the problem. + ## Markdown Output LiteParse can render documents directly to Markdown including headings, tables, lists, @@ -55,6 +77,8 @@ const parser = new LiteParse({ tessdataPath: undefined, // Path to tessdata directory (optional) maxPages: 1000, // Max pages to parse targetPages: '1-5,10', // Specific pages (optional) + extractScreenshots: false, // Return parsed pages as PNG buffers + continueOnPageError: false, // Skip broken pages and return pageErrors dpi: 150, // Rendering DPI outputFormat: 'json', // "json" | "text" | "markdown" imageMode: 'placeholder', // Markdown image handling: "placeholder" | "off" | "embed" @@ -101,7 +125,12 @@ title, typed attributes, MCIDs, children, and referenced link annotations. Untag pages have an empty `roots` array; the field is omitted when disabled. Every result also carries the document's `/Info` `creator`/`producer` when -present (API-level only, not in CLI JSON), and with `extractContentBounds` +present. With `extractDocumentMetadata`, `result.docMeta` adds a provenance +object with dates, PDF version/security, signature state, incremental-save +markers, trailer ID comparison, the catalog's XMP packet (capped at 64 KiB; +skipped for sources over 16 MiB), and source size — off by default since it +streams the whole file, and absent for inputs converted from a non-PDF +format. These are API-level only, not in CLI JSON. With `extractContentBounds` each page carries a `contentBounds` union bbox of its top-level content objects. With `extractXfaPackets`, `result.xfaPackets` lists each raw XFA packet (index, name, content length, XML content); non-XFA documents yield an diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts index ad9ad0eb..c7eb8d24 100644 --- a/packages/node/native.d.ts +++ b/packages/node/native.d.ts @@ -21,6 +21,16 @@ export interface JsLiteParseConfig { maxPages?: number /** Specific pages to parse (e.g., "1-5,10,15-20"). */ targetPages?: string + /** + * Render parsed pages to PNG and return them in `ParseResult.screenshots`. + * Default false; PNG payloads can be large. + */ + extractScreenshots?: boolean + /** + * Continue after page-level extraction failures and return them in + * `ParseResult.pageErrors`. Default false. + */ + continueOnPageError?: boolean /** DPI for rendering pages (used for OCR and screenshots). */ dpi?: number /** Output format: "json", "text", or "markdown". */ @@ -51,6 +61,11 @@ export interface JsLiteParseConfig { * (default true). Set false for plain anchor text. */ extractLinks?: boolean + /** + * Keep running headers/footers in markdown output instead of stripping + * repeated page-band lines and page chrome (default false). + */ + keepHeadersFooters?: boolean /** Extract all PDF annotations as page-scoped structured data. */ extractAnnotations?: boolean /** Extract AcroForm widget fields and values. */ @@ -62,6 +77,12 @@ export interface JsLiteParseConfig { * `ParseResult.xfaPackets`. Default false. */ extractXfaPackets?: boolean + /** + * Collect document provenance metadata into `ParseResult.docMeta`. + * Default false: it streams the whole source file once. Absent for + * inputs converted from a non-PDF format. + */ + extractDocumentMetadata?: boolean /** * Emit each page's `contentBounds` (union bbox of top-level content * objects, viewport coords). Default false. @@ -72,7 +93,10 @@ export interface JsLiteParseConfig { * them to each screenshot result. Default false. */ detectScreenshotRects?: boolean - /** Draw AcroForm field appearances into rendered rasters (runs document open/JS actions). Default false. */ + /** + * Draw AcroForm field appearances into rendered rasters (screenshots and + * OCR inputs). Runs the document's open/JS actions. Default false. + */ renderFormFields?: boolean /** * Whether a systemic OCR failure aborts the whole parse (default true). @@ -155,6 +179,7 @@ export interface JsTextItem { charCodes?: Array /** True when the trailing source space was synthesized by PDFium. */ trailingSpaceGenerated?: boolean + /** OCR confidence score (0.0-1.0). Undefined for native PDF text. */ confidence?: number /** Rotation in degrees (viewport space). Defaults to 0 when omitted. */ rotation?: number @@ -310,18 +335,57 @@ export interface JsFormField { selectedOptions: Array } export interface JsParseResult { + /** Total source-document pages before target/max-page filtering. */ + totalPages: number pages: Array + pageErrors: Array text: string images: Array + screenshots: Array imageErrorCount: number formType?: number /** The document's `/Info` `Creator` entry, when present. */ creator?: string /** The document's `/Info` `Producer` entry, when present. */ producer?: string + /** + * Document-level provenance metadata; present only when + * `extractDocumentMetadata` is enabled and the input was a real PDF. + */ + docMeta?: JsDocumentMetadata /** Raw XFA packets; present only when `extractXfaPackets` is enabled. */ xfaPackets?: Array } +export interface JsPageError { + pageNum: number + message: string +} +/** One batch of pages from a `ParseSession`. */ +export interface JsParseBatch { + /** First source page in this batch, 1-indexed. */ + startPage: number + /** Last source page in this batch, 1-indexed and inclusive. */ + endPage: number + /** The pages in `startPage..=endPage`, as an ordinary parse result. */ + result: JsParseResult +} +export interface JsDocumentMetadata { + creationDate?: string + modDate?: string + fileVersion?: number + isEncrypted?: boolean + securityHandlerRevision?: number + permissions?: number + eofSectionCount?: number + startxrefCount?: number + trailerIdPairDiffers?: boolean + rawFileSize?: number + xmp?: string + /** True when the catalog's XMP stream exceeded the 64 KiB cap. */ + xmpTruncated?: boolean + signatureCount?: number + signatureByteRangeReachesEof?: boolean +} /** One raw packet from an XFA form document's `/XFA` array. */ export interface JsXfaPacket { index: number @@ -391,9 +455,29 @@ export interface JsPageComplexityStats { textLength: number textCoverage: number hasSubstantialImages: boolean + /** + * Number of counted raster images — inline figures only; full-page + * backgrounds are excluded (see `fullPageImage`). + */ imageBlockCount: number + /** + * Summed image-bbox area over page area, clamped to 1. Counts inline + * figures only: a full-page scan raster contributes 0 here — check + * `fullPageImage` for that. + */ imageCoverage: number + /** + * Largest single counted image's area over page area, clamped to 1. Same + * exclusion as `imageCoverage`: a full-page raster contributes 0. + */ largestImageCoverage: number + /** + * A single raster covering ≥90% of the page is present. Such full-page + * backgrounds are excluded from `imageCoverage`/`largestImageCoverage` + * (they're not inline figures), so this flag is the only signal that + * distinguishes a scan from a genuinely blank page — both otherwise + * report no text and no counted images. + */ fullPageImage: boolean uncoveredVectorArea?: number isGarbled: boolean @@ -413,6 +497,17 @@ export declare class LiteParse { constructor(config?: JsLiteParseConfig | undefined | null) /** Parse a document. Accepts a file path (string) or raw PDF bytes (Buffer). */ parse(input: string | Buffer): Promise + /** + * Open a document for bounded-memory batch parsing. Internal plumbing + * for the JS wrapper's `parseBatches()` — prefer that; it also closes + * the session for you. + * + * Converts a non-PDF source once, then yields `batchSize` pages at a time + * via `nextBatch()` (default 25). Cross-page passes (repeated + * header/footer removal, image deduplication) see only the pages in their + * own batch, so output can differ from a whole-document `parse()`. + */ + openBatchSession(input: string | Buffer, batchSize?: number | undefined | null): Promise /** * Parse from pre-extracted pages, skipping PDFium text extraction. * @@ -440,3 +535,28 @@ export declare class LiteParse { /** Get the current configuration. */ get config(): JsLiteParseConfig } +/** + * A document opened once and parsed in bounded page batches. Internal + * plumbing for the JS wrapper's `parseBatches()` — prefer that. + * + * Created by `LiteParse.openBatchSession()`. The converted-PDF temporary + * file for a non-PDF source lives as long as the session, so conversion is + * paid once no matter how many batches are consumed. Call `close()` when + * abandoning the session early — otherwise that temp file waits for GC. + */ +export declare class ParseSession { + /** Total pages in the source document, before `maxPages` or batching. */ + get totalPages(): number + /** + * Parse and return the next batch, or `null` once every page within + * `maxPages` has been yielded. Rejects if the session is closed. + */ + nextBatch(): Promise + /** + * Release the session's resources now — most importantly the converted + * temporary PDF for a non-PDF source, which otherwise lives until the + * JS object is garbage collected. Idempotent; `nextBatch()` rejects + * afterwards. + */ + close(): Promise +} diff --git a/packages/node/package.json b/packages/node/package.json index 64ebd220..b5a2f7b1 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@llamaindex/liteparse", - "version": "2.10.1", + "version": "2.12.0", "description": "Fast, lightweight PDF and document parsing with spatial text extraction", "type": "module", "main": "./dist/lib.js", @@ -54,13 +54,13 @@ "typescript": "~5.9.2" }, "optionalDependencies": { - "@llamaindex/liteparse-darwin-arm64": "2.10.1", - "@llamaindex/liteparse-darwin-x64": "2.10.1", - "@llamaindex/liteparse-linux-arm64-gnu": "2.10.1", - "@llamaindex/liteparse-linux-x64-gnu": "2.10.1", - "@llamaindex/liteparse-win32-arm64-msvc": "2.10.1", - "@llamaindex/liteparse-win32-x64-msvc": "2.10.1", - "@llamaindex/liteparse-linux-x64-musl": "2.10.1" + "@llamaindex/liteparse-darwin-arm64": "2.12.0", + "@llamaindex/liteparse-darwin-x64": "2.12.0", + "@llamaindex/liteparse-linux-arm64-gnu": "2.12.0", + "@llamaindex/liteparse-linux-x64-gnu": "2.12.0", + "@llamaindex/liteparse-win32-arm64-msvc": "2.12.0", + "@llamaindex/liteparse-win32-x64-msvc": "2.12.0", + "@llamaindex/liteparse-linux-x64-musl": "2.12.0" }, "engines": { "node": ">=18.0.0" @@ -79,4 +79,4 @@ }, "author": "LlamaIndex", "license": "Apache-2.0" -} \ No newline at end of file +} diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.ts index 3b2a7e8b..2e0b8a44 100644 --- a/packages/node/src/cli.ts +++ b/packages/node/src/cli.ts @@ -93,6 +93,10 @@ program "--target-pages ", 'Pages to parse (e.g., "1-5,10,15-20")', ) + .option( + "--continue-on-page-error", + "Continue after page-level extraction errors and report them in JSON", + ) .option("--dpi ", "Rendering DPI", parseFloat) .option("--preserve-small-text", "Keep very small text") .option( @@ -145,6 +149,7 @@ program if (opts.ocrLanguage) config.ocrLanguage = opts.ocrLanguage as string; if (opts.maxPages) config.maxPages = opts.maxPages as number; if (opts.targetPages) config.targetPages = opts.targetPages as string; + if (opts.continueOnPageError) config.continueOnPageError = true; if (opts.dpi) config.dpi = opts.dpi as number; if (opts.preserveSmallText) config.preserveVerySmallText = true; if (opts.extractTextMetadata) config.extractTextMetadata = true; @@ -160,6 +165,14 @@ program const parser = new LiteParse(config); const result = await parser.parse(await resolveInput(file)); + // JSON output carries pageErrors itself; text/markdown would silently + // omit the failed pages, so always surface them on stderr. + for (const error of result.pageErrors) { + console.error( + `[liteparse] page ${error.pageNum} failed to extract and was skipped: ${error.message}`, + ); + } + const output = config.outputFormat === "json" ? JSON.stringify( diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts index 95ec3190..21df191e 100644 --- a/packages/node/src/lib.ts +++ b/packages/node/src/lib.ts @@ -9,6 +9,7 @@ import { type NativeExtractedImage, type NativeStructureTreeElement, type NativePageComplexityStats, + type NativeScreenshotResult, } from "./native.js"; // --------------------------------------------------------------------------- @@ -28,6 +29,10 @@ export interface LiteParseConfig { tessdataPath?: string; maxPages: number; targetPages?: string; + /** Render parsed pages to PNG and return them in `ParseResult.screenshots`. */ + extractScreenshots: boolean; + /** Continue after page-level extraction failures and collect `pageErrors`. */ + continueOnPageError: boolean; dpi: number; outputFormat: OutputFormat; /** How to surface raster images in markdown output (default: "placeholder"). */ @@ -48,6 +53,11 @@ export interface LiteParseConfig { extractStructureTree: boolean; /** Extract raw XFA packets (name + XML content) into `ParseResult.xfaPackets` (default: false). */ extractXfaPackets: boolean; + /** + * Collect document provenance metadata into `result.docMeta`. Default + * false: Absent for inputs converted from a non-PDF format. + */ + extractDocumentMetadata: boolean; /** Emit each page's `contentBounds` (union bbox of top-level content objects) (default: false). */ extractContentBounds: boolean; /** Detect solid rectangles/lines in rendered page screenshots (default: false). */ @@ -342,10 +352,16 @@ export interface ExtractedImage { } export interface ParseResult { + /** Total source-document pages before `targetPages` or `maxPages` filtering. */ + totalPages: number; pages: ParsedPage[]; + /** Page-level PDFium extraction failures when tolerance is enabled. */ + pageErrors: Array<{ pageNum: number; message: string }>; text: string; /** Populated only when `extractImages` is true. */ images: ExtractedImage[]; + /** PNG screenshots of parsed pages when `extractScreenshots` is enabled. */ + screenshots: ScreenshotResult[]; /** Embedded image objects that PDFium could not render or encode. */ imageErrorCount: number; /** PDFium form type, present only when `extractFormFields` is enabled. */ @@ -354,10 +370,57 @@ export interface ParseResult { creator?: string; /** The document's `/Info` `Producer` entry, when present. */ producer?: string; + /** + * Document-level provenance metadata from PDFium and the source PDF. + * Present only when `extractDocumentMetadata` is enabled and the input was + * a real PDF (not converted from DOCX/XLSX/an image). + */ + docMeta?: DocumentMetadata; /** Raw XFA packets; present only when `extractXfaPackets` is enabled. */ xfaPackets?: XfaPacket[]; } +export interface ParseBatchOptions { + /** Pages materialized in one batch. Default: 25. */ + batchSize?: number; +} + +export interface ParseBatch { + /** First source page in this batch (1-indexed). */ + startPage: number; + /** Last source page in this batch (1-indexed, inclusive). */ + endPage: number; + /** Total source-document pages, before the parser's `maxPages` cap. */ + totalPages: number; + result: ParseResult; +} + +/** Provenance and tamper-analysis facts extracted from the source PDF. */ +export interface DocumentMetadata { + creationDate?: string; + modDate?: string; + /** Encoded PDF version (`14` means PDF 1.4). */ + fileVersion?: number; + isEncrypted?: boolean; + securityHandlerRevision?: number; + permissions?: number; + eofSectionCount?: number; + startxrefCount?: number; + trailerIdPairDiffers?: boolean; + rawFileSize?: number; + /** + * The document catalog's `/Metadata` XMP packet, capped at 64 KiB. Absent + * when the document has none, when it is too large to resolve cheaply, or + * in WASM builds. + */ + xmp?: string; + /** True when the catalog's XMP stream exceeded the 64 KiB cap. */ + xmpTruncated?: boolean; + signatureCount?: number; + /** False when bytes were appended after a readable signature byte range. */ + signatureByteRangeReachesEof?: boolean; +} + /** One raw packet from an XFA form document's `/XFA` array. */ export interface XfaPacket { index: number; @@ -494,6 +557,8 @@ export class LiteParse { tessdataPath: userConfig.tessdataPath, maxPages: userConfig.maxPages, targetPages: userConfig.targetPages, + extractScreenshots: userConfig.extractScreenshots, + continueOnPageError: userConfig.continueOnPageError, dpi: userConfig.dpi, outputFormat: userConfig.outputFormat, imageMode: userConfig.imageMode, @@ -505,6 +570,7 @@ export class LiteParse { extractFormFields: userConfig.extractFormFields, extractStructureTree: userConfig.extractStructureTree, extractXfaPackets: userConfig.extractXfaPackets, + extractDocumentMetadata: userConfig.extractDocumentMetadata, extractContentBounds: userConfig.extractContentBounds, detectScreenshotRects: userConfig.detectScreenshotRects, renderFormFields: userConfig.renderFormFields, @@ -534,6 +600,8 @@ export class LiteParse { tessdataPath: resolved.tessdataPath ?? undefined, maxPages: resolved.maxPages ?? 1000, targetPages: resolved.targetPages ?? undefined, + extractScreenshots: resolved.extractScreenshots ?? false, + continueOnPageError: resolved.continueOnPageError ?? false, dpi: resolved.dpi ?? 150, outputFormat: (resolved.outputFormat as OutputFormat) ?? "json", imageMode: (resolved.imageMode as ImageMode) ?? "placeholder", @@ -545,6 +613,7 @@ export class LiteParse { extractFormFields: resolved.extractFormFields ?? false, extractStructureTree: resolved.extractStructureTree ?? false, extractXfaPackets: resolved.extractXfaPackets ?? false, + extractDocumentMetadata: resolved.extractDocumentMetadata ?? false, extractContentBounds: resolved.extractContentBounds ?? false, detectScreenshotRects: resolved.detectScreenshotRects ?? false, renderFormFields: resolved.renderFormFields ?? false, @@ -568,16 +637,59 @@ export class LiteParse { const nativeInput = typeof input === "string" ? input : Buffer.from(input); const result: NativeParseResult = await this._native.parse(nativeInput); - return { - pages: result.pages.map(toPage), - text: result.text, - images: (result.images ?? []).map(toImage), - imageErrorCount: result.imageErrorCount ?? 0, - formType: result.formType, - creator: result.creator, - producer: result.producer, - xfaPackets: result.xfaPackets, - }; + return toParseResult(result); + } + + /** + * Parse a document in bounded-memory page batches of `batchSize` pages. + * + * Each yielded result is independent and becomes collectible once the caller + * advances the iterator, so a consumer that does not retain batches never + * holds more than one batch of pages in memory. A non-PDF source is + * converted once when the iterator starts, not once per batch; its temporary + * file is released when iteration ends — including an early `break` or + * `throw`, which run the generator's cleanup. + * + * Cross-page passes see only the pages in their own batch, so repeated + * header/footer removal and image deduplication are batch-local and the + * output can differ from `parse()`. Prefer `parse()` unless the size of the + * materialized result is the problem. + * + * As with any async generator, work starts on the first `next()` call, so + * errors (an unreadable file, or a parser configured with `targetPages` — + * ambiguous with generated batch ranges) surface on the first iteration + * rather than when `parseBatches()` itself is called. + */ + async *parseBatches( + input: LiteParseInput, + options: ParseBatchOptions = {}, + ): AsyncGenerator { + const nativeInput = typeof input === "string" ? input : Buffer.from(input); + const session = await this._native.openBatchSession( + nativeInput, + options.batchSize, + ); + try { + const totalPages = session.totalPages; + + for (;;) { + const batch = await session.nextBatch(); + if (batch == null) { + return; + } + yield { + startPage: batch.startPage, + endPage: batch.endPage, + totalPages, + result: toParseResult(batch.result), + }; + } + } finally { + // Frees the session's converted-PDF temp file now instead of at GC — + // this runs on normal exhaustion and when the consumer abandons the + // loop early. + await session.close(); + } } /** @@ -595,12 +707,7 @@ export class LiteParse { graphics: p.graphics, })); const result = this._native.parsePages(nativePages); - return { - pages: result.pages.map(toPage), - text: result.text, - images: (result.images ?? []).map(toImage), - imageErrorCount: result.imageErrorCount ?? 0, - }; + return toParseResult(result); } /** @@ -671,6 +778,23 @@ function toComplexity(s: NativePageComplexityStats): PageComplexityStats { }; } +function toParseResult(result: NativeParseResult): ParseResult { + return { + totalPages: result.totalPages, + pages: result.pages.map(toPage), + pageErrors: result.pageErrors ?? [], + text: result.text, + images: (result.images ?? []).map(toImage), + screenshots: (result.screenshots ?? []).map(toScreenshot), + imageErrorCount: result.imageErrorCount ?? 0, + formType: result.formType, + creator: result.creator, + producer: result.producer, + docMeta: result.docMeta, + xfaPackets: result.xfaPackets, + }; +} + function toPage(p: NativeParsedPage): ParsedPage { return { pageNum: p.pageNum, @@ -750,6 +874,17 @@ function toImage(img: NativeExtractedImage): ExtractedImage { }; } +function toScreenshot(result: NativeScreenshotResult): ScreenshotResult { + return { + pageNum: result.pageNum, + width: result.width, + height: result.height, + imageBuffer: result.imageBuffer, + isSolidFill: result.isSolidFill, + rects: result.rects, + }; +} + function toTextItem(item: NativeTextItem): TextItem { return { text: item.text, diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts index 247ceebe..4991dfb8 100644 --- a/packages/node/src/native.ts +++ b/packages/node/src/native.ts @@ -27,6 +27,8 @@ export interface LiteParseNativeConfig { tessdataPath?: string; maxPages?: number; targetPages?: string; + extractScreenshots?: boolean; + continueOnPageError?: boolean; dpi?: number; outputFormat?: string; imageMode?: string; @@ -38,6 +40,7 @@ export interface LiteParseNativeConfig { extractFormFields?: boolean; extractStructureTree?: boolean; extractXfaPackets?: boolean; + extractDocumentMetadata?: boolean; extractContentBounds?: boolean; detectScreenshotRects?: boolean; renderFormFields?: boolean; @@ -233,16 +236,37 @@ export interface NativeExtractedImage { } export interface NativeParseResult { + totalPages: number; pages: NativeParsedPage[]; + pageErrors: Array<{ pageNum: number; message: string }>; text: string; images: NativeExtractedImage[]; + screenshots: NativeScreenshotResult[]; imageErrorCount: number; formType?: number; creator?: string; producer?: string; + docMeta?: NativeDocumentMetadata; xfaPackets?: NativeXfaPacket[]; } +export interface NativeDocumentMetadata { + creationDate?: string; + modDate?: string; + fileVersion?: number; + isEncrypted?: boolean; + securityHandlerRevision?: number; + permissions?: number; + eofSectionCount?: number; + startxrefCount?: number; + trailerIdPairDiffers?: boolean; + rawFileSize?: number; + xmp?: string; + xmpTruncated?: boolean; + signatureCount?: number; + signatureByteRangeReachesEof?: boolean; +} + export interface NativeXfaPacket { index: number; name?: string; @@ -296,8 +320,24 @@ export interface NativePageComplexityStats { layout?: NativeLayoutComplexityStats; } +export interface NativeParseBatch { + startPage: number; + endPage: number; + result: NativeParseResult; +} + +export interface NativeParseSession { + nextBatch(): Promise; + close(): Promise; + readonly totalPages: number; +} + export interface LiteParseNative { parse(input: string | Buffer): Promise; + openBatchSession( + input: string | Buffer, + batchSize?: number, + ): Promise; parsePages(pages: NativePageInput[]): NativeParseResult; isComplex(input: string | Buffer): Promise; screenshot( diff --git a/packages/python/README.md b/packages/python/README.md index 434b5f2a..0517bd7c 100644 --- a/packages/python/README.md +++ b/packages/python/README.md @@ -18,6 +18,7 @@ from liteparse import LiteParse parser = LiteParse() result = parser.parse("document.pdf") print(result.text) +print(f"Source document pages: {result.total_pages}") # Access structured data for page in result.pages: @@ -54,6 +55,8 @@ parser = LiteParse( tessdata_path=None, # Path to tessdata directory (optional) max_pages=1000, # Max pages to parse target_pages="1-5,10", # Specific pages (optional) + extract_screenshots=False, # Return parsed pages as PNG bytes + continue_on_page_error=False, # Skip broken pages and return page_errors dpi=150, # Rendering DPI output_format="json", # "json" | "text" | "markdown" image_mode="placeholder", # Markdown image handling: "placeholder" | "off" | "embed" @@ -93,6 +96,14 @@ all tagged-PDF roots and recursive elements with type, ID, actual/alternate text title, typed attributes, MCIDs, children, and referenced link annotations. Untagged pages have an empty ``roots`` list; the field is ``None`` when disabled. +Every result also carries ``creator``/``producer`` from the PDF ``/Info`` +dictionary. With ``extract_document_metadata=True``, ``result.doc_meta`` adds a +provenance object with dates, PDF version/security, signature state, +incremental-save markers, trailer ID comparison, the catalog's XMP packet +(capped at 64 KiB; skipped for sources over 16 MiB), and source size. It is off by default because it streams the whole source file, +and it is ``None`` for inputs converted from a non-PDF format. These document +fields are API-only and do not alter default CLI JSON. + ## Parsing from Bytes Pass raw PDF bytes directly — useful for web uploads or downloaded files: diff --git a/packages/python/liteparse/__init__.py b/packages/python/liteparse/__init__.py index c0697262..483bde53 100644 --- a/packages/python/liteparse/__init__.py +++ b/packages/python/liteparse/__init__.py @@ -12,7 +12,10 @@ LayoutComplexityStats, LiteParseConfig, PageComplexityStats, + PageError, ParseResult, + ParseBatch, + DocumentMetadata, XfaPacket, ParsedPage, TextItem, @@ -38,6 +41,9 @@ "StructureTreeElement", "LiteParseConfig", "ParseResult", + "PageError", + "ParseBatch", + "DocumentMetadata", "XfaPacket", "ParsedPage", "TextItem", diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py index 95a361aa..a3a2aaf2 100644 --- a/packages/python/liteparse/parser.py +++ b/packages/python/liteparse/parser.py @@ -1,7 +1,7 @@ """LiteParse Python wrapper - native Rust bindings via PyO3.""" from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union from liteparse._liteparse import LiteParse as _NativeLiteParse from liteparse._liteparse import search_items as _native_search_items @@ -18,8 +18,11 @@ LiteParseConfig, PageComplexityStats, ParsedPage, + ParseBatch, ParseError, + PageError, ParseResult, + DocumentMetadata, ScreenshotRect, ScreenshotResult, TextItem, @@ -308,14 +311,68 @@ def _convert_native_result(native_result: Any) -> ParseResult: for img in getattr(native_result, "images", []) ] native_xfa_packets = getattr(native_result, "xfa_packets", None) + native_doc_meta = getattr(native_result, "doc_meta", None) + doc_meta = ( + None + if native_doc_meta is None + else DocumentMetadata( + creation_date=getattr(native_doc_meta, "creation_date", None), + mod_date=getattr(native_doc_meta, "mod_date", None), + file_version=getattr(native_doc_meta, "file_version", None), + is_encrypted=getattr(native_doc_meta, "is_encrypted", None), + security_handler_revision=getattr( + native_doc_meta, "security_handler_revision", None + ), + permissions=getattr(native_doc_meta, "permissions", None), + eof_section_count=getattr(native_doc_meta, "eof_section_count", None), + startxref_count=getattr(native_doc_meta, "startxref_count", None), + trailer_id_pair_differs=getattr( + native_doc_meta, "trailer_id_pair_differs", None + ), + raw_file_size=getattr(native_doc_meta, "raw_file_size", None), + xmp=getattr(native_doc_meta, "xmp", None), + xmp_truncated=getattr(native_doc_meta, "xmp_truncated", None), + signature_count=getattr(native_doc_meta, "signature_count", None), + signature_byte_range_reaches_eof=getattr( + native_doc_meta, "signature_byte_range_reaches_eof", None + ), + ) + ) return ParseResult( pages=pages, text=native_result.text, + total_pages=getattr(native_result, "total_pages", len(pages)), images=images, + screenshots=[ + ScreenshotResult( + page_num=screenshot.page_num, + width=screenshot.width, + height=screenshot.height, + image_bytes=screenshot.image_bytes, + is_solid_fill=getattr(screenshot, "is_solid_fill", False), + rects=[ + ScreenshotRect( + x=rect.x, + y=rect.y, + width=rect.width, + height=rect.height, + color=rect.color, + is_line=rect.is_line, + ) + for rect in getattr(screenshot, "rects", []) + ], + ) + for screenshot in getattr(native_result, "screenshots", []) + ], image_error_count=getattr(native_result, "image_error_count", 0), + page_errors=[ + PageError(page_num=error.page_num, message=error.message) + for error in getattr(native_result, "page_errors", []) + ], form_type=getattr(native_result, "form_type", None), creator=getattr(native_result, "creator", None), producer=getattr(native_result, "producer", None), + doc_meta=doc_meta, xfa_packets=( [ XfaPacket( @@ -355,6 +412,8 @@ def __init__( tessdata_path: Optional[str] = None, max_pages: Optional[int] = None, target_pages: Optional[str] = None, + extract_screenshots: Optional[bool] = None, + continue_on_page_error: Optional[bool] = None, dpi: Optional[float] = None, output_format: Optional[str] = None, preserve_very_small_text: Optional[bool] = None, @@ -370,6 +429,7 @@ def __init__( extract_form_fields: Optional[bool] = None, extract_structure_tree: Optional[bool] = None, extract_xfa_packets: Optional[bool] = None, + extract_document_metadata: Optional[bool] = None, extract_content_bounds: Optional[bool] = None, detect_screenshot_rects: Optional[bool] = None, render_form_fields: Optional[bool] = None, @@ -394,6 +454,11 @@ def __init__( tessdata_path: Path to tessdata directory for Tesseract max_pages: Maximum number of pages to parse target_pages: Specific pages to parse (e.g., "1-5,10,15-20") + extract_screenshots: Render parsed pages to PNG and return them in + ``ParseResult.screenshots``. Default False; PNG payloads can be large. + continue_on_page_error: Skip page-level PDF extraction failures and + return them in ``ParseResult.page_errors``. Document-level + failures remain fatal. Default False. dpi: DPI for rendering (affects OCR quality) output_format: Output format: "json", "text", or "markdown" (default: "json") preserve_very_small_text: Whether to preserve very small text @@ -467,6 +532,10 @@ def __init__( kwargs["max_pages"] = max_pages if target_pages is not None: kwargs["target_pages"] = target_pages + if extract_screenshots is not None: + kwargs["extract_screenshots"] = extract_screenshots + if continue_on_page_error is not None: + kwargs["continue_on_page_error"] = continue_on_page_error if dpi is not None: kwargs["dpi"] = dpi if output_format is not None: @@ -497,6 +566,8 @@ def __init__( kwargs["extract_structure_tree"] = extract_structure_tree if extract_xfa_packets is not None: kwargs["extract_xfa_packets"] = extract_xfa_packets + if extract_document_metadata is not None: + kwargs["extract_document_metadata"] = extract_document_metadata if extract_content_bounds is not None: kwargs["extract_content_bounds"] = extract_content_bounds if detect_screenshot_rects is not None: @@ -553,6 +624,78 @@ def parse( except Exception as e: raise ParseError(str(e)) from e + def parse_batches( + self, + file_data: Union[str, Path, bytes], + batch_size: Optional[int] = None, + ) -> Iterator[ParseBatch]: + """ + Parse a document in bounded-memory page batches. + + Each yielded batch is an ordinary :class:`ParseResult` covering + ``batch.start_page`` through ``batch.end_page``, and becomes + collectible as soon as you advance the iterator — so a loop that does + not retain batches never holds more than one batch of pages in memory. + A non-PDF source is converted once, not once per batch. + + Cross-page passes see only the pages in their own batch, so repeated + header/footer removal and image deduplication are batch-local and the + output can differ from :meth:`parse`. Prefer :meth:`parse` unless the + size of the materialized result is the problem. + + Args: + file_data: Path to the document file, or raw PDF bytes. + batch_size: Pages materialized per batch (default 25). + + Yields: + ParseBatch for each page range, in document order. + + Raises: + ParseError: If parsing fails, or if the parser was constructed + with ``target_pages`` (ambiguous with generated batch ranges). + Open errors are raised here, when ``parse_batches`` is called; + per-batch parse errors are raised from the iterator. + FileNotFoundError: If the file doesn't exist. + """ + # Validate and open eagerly — this is not a generator function, so a + # missing file or a target_pages conflict raises here rather than on + # the first iteration of the returned iterator. + try: + if isinstance(file_data, bytes): + session = self._native.open_batch_session_bytes( + file_data, batch_size + ) + else: + file_path = Path(file_data) + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + session = self._native.open_batch_session( + str(file_path.absolute()), batch_size + ) + except FileNotFoundError: + raise + except Exception as e: + raise ParseError(str(e)) from e + + return self._iter_batches(session) + + @staticmethod + def _iter_batches(session: Any) -> Iterator[ParseBatch]: + total_pages = session.total_pages + while True: + try: + batch = session.next_batch() + except Exception as e: + raise ParseError(str(e)) from e + if batch is None: + return + yield ParseBatch( + start_page=batch.start_page, + end_page=batch.end_page, + total_pages=total_pages, + result=_convert_native_result(batch.result), + ) + def is_complex( self, file_data: Union[str, Path, bytes], @@ -657,6 +800,8 @@ def get_config(self) -> LiteParseConfig: tessdata_path=cfg.tessdata_path, max_pages=cfg.max_pages, target_pages=cfg.target_pages, + extract_screenshots=cfg.extract_screenshots, + continue_on_page_error=cfg.continue_on_page_error, dpi=cfg.dpi, output_format=cfg.output_format, preserve_very_small_text=cfg.preserve_very_small_text, @@ -680,6 +825,7 @@ def get_config(self) -> LiteParseConfig: extract_images=cfg.extract_images, extract_vector_graphics=cfg.extract_vector_graphics, extract_xfa_packets=cfg.extract_xfa_packets, + extract_document_metadata=cfg.extract_document_metadata, extract_content_bounds=cfg.extract_content_bounds, detect_screenshot_rects=cfg.detect_screenshot_rects, ) diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py index 6e02a154..9705fa94 100644 --- a/packages/python/liteparse/types.py +++ b/packages/python/liteparse/types.py @@ -219,19 +219,58 @@ class XfaPacket: content: Optional[str] +@dataclass +class DocumentMetadata: + """Document-level provenance metadata from PDFium and the source PDF.""" + creation_date: Optional[str] = None + mod_date: Optional[str] = None + #: Encoded PDF version (14 means PDF 1.4). + file_version: Optional[int] = None + is_encrypted: Optional[bool] = None + security_handler_revision: Optional[int] = None + permissions: Optional[int] = None + eof_section_count: Optional[int] = None + startxref_count: Optional[int] = None + trailer_id_pair_differs: Optional[bool] = None + raw_file_size: Optional[int] = None + #: The document catalog's ``/Metadata`` XMP packet, capped at 64 KiB. + #: ``None`` when the document has none or it is too large to resolve + #: cheaply. + xmp: Optional[str] = None + #: True when the catalog's XMP stream exceeded the 64 KiB cap. + xmp_truncated: Optional[bool] = None + signature_count: Optional[int] = None + signature_byte_range_reaches_eof: Optional[bool] = None + + +@dataclass +class PageError: + """A page-level extraction failure skipped during a tolerant parse.""" + page_num: int + message: str + + @dataclass class ParseResult: """Result of parsing a document.""" pages: List[ParsedPage] text: str + #: Total source-document pages before target/max-page filtering. + total_pages: int = 0 images: List[ExtractedImage] = field(default_factory=list) + screenshots: List["ScreenshotResult"] = field(default_factory=list) image_error_count: int = 0 + page_errors: List[PageError] = field(default_factory=list) #: PDFium form type, present only when ``extract_form_fields=True``. form_type: Optional[int] = None #: The document's ``/Info`` ``Creator`` entry, when present. creator: Optional[str] = None #: The document's ``/Info`` ``Producer`` entry, when present. producer: Optional[str] = None + #: Document-level provenance metadata. Present only when + #: ``extract_document_metadata=True`` and the input was a real PDF + #: (not converted from DOCX/XLSX/an image). + doc_meta: Optional[DocumentMetadata] = None #: Raw XFA packets; present only when ``extract_xfa_packets=True``. xfa_packets: Optional[List[XfaPacket]] = None @@ -247,6 +286,19 @@ def get_page(self, page_num: int) -> Optional[ParsedPage]: return None +@dataclass +class ParseBatch: + """One batch of pages from :meth:`LiteParse.parse_batches`.""" + #: First source page in this batch (1-indexed). + start_page: int + #: Last source page in this batch (1-indexed, inclusive). + end_page: int + #: Total source-document pages, before the parser's ``max_pages`` cap. + total_pages: int + #: The pages in ``start_page..end_page``, as an ordinary parse result. + result: ParseResult + + @dataclass class ScreenshotRect: """One solid rectangle (or line) detected in a rendered page bitmap, @@ -354,6 +406,8 @@ class LiteParseConfig: tessdata_path: Optional[str] max_pages: int target_pages: Optional[str] + extract_screenshots: bool + continue_on_page_error: bool dpi: float output_format: str preserve_very_small_text: bool @@ -381,6 +435,7 @@ class LiteParseConfig: extract_images: bool = False extract_vector_graphics: bool = False extract_xfa_packets: bool = False + extract_document_metadata: bool = False detect_screenshot_rects: bool = False extract_content_bounds: bool = False diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml index 48032d8b..0774ae5b 100644 --- a/packages/python/pyproject.toml +++ b/packages/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "liteparse" -version = "2.10.1" +version = "2.12.0" description = "Python bindings for LiteParse - fast, lightweight PDF and document parsing" readme = "README.md" license = "Apache-2.0" diff --git a/packages/python/uv.lock b/packages/python/uv.lock index b5308fa1..215882b2 100644 --- a/packages/python/uv.lock +++ b/packages/python/uv.lock @@ -128,7 +128,7 @@ wheels = [ [[package]] name = "liteparse" -version = "2.0.5" +version = "2.11.1" source = { editable = "." } [package.optional-dependencies] diff --git a/packages/wasm/README.md b/packages/wasm/README.md index 503e3f4f..5762454d 100644 --- a/packages/wasm/README.md +++ b/packages/wasm/README.md @@ -64,6 +64,7 @@ All optional, camelCase: | `ocrEnabled` | `boolean` | `true` | Run OCR on text-sparse pages | | `maxPages` | `number` | `1000` | Stop after this many pages | | `targetPages` | `string` | — | e.g. `"1-5,10,15-20"` | +| `extractScreenshots` | `boolean` | `false` | Return parsed pages as PNG bytes on `result.screenshots` | | `dpi` | `number` | `150` | Render DPI for OCR / screenshots | | `outputFormat` | `"json" \| "text" \| "markdown"` | `"json"` | Output format; `"markdown"` returns rendered Markdown on `result.text` | | `imageMode` | `"off" \| "placeholder" \| "embed"` | `"placeholder"` | How raster images are surfaced in markdown output | diff --git a/packages/wasm/package.json b/packages/wasm/package.json index 552405f4..e5d0a8e8 100644 --- a/packages/wasm/package.json +++ b/packages/wasm/package.json @@ -1,6 +1,6 @@ { "name": "@llamaindex/liteparse-wasm", - "version": "2.10.1", + "version": "2.12.0", "description": "Fast, lightweight PDF parsing with spatial text extraction — WebAssembly build for browsers", "type": "module", "main": "./pkg/liteparse_wasm.js", diff --git a/scripts/generate_filled_acroform_fixture.py b/scripts/generate_filled_acroform_fixture.py new file mode 100644 index 00000000..22143418 --- /dev/null +++ b/scripts/generate_filled_acroform_fixture.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Generate `integration_tests_data/filled_acroform.pdf`. + +The fixture backs `test_filled_acroform_values_are_extracted_as_text`, which +asserts exact annotation and field counts. Writing the PDF by hand (rather than +checking in an opaque blob from some authoring tool) keeps those numbers +auditable and lets each widget target one specific behaviour: + +Page 1 — 6 annotations, 5 form fields + 1. `customer_name` value painted directly by the widget's /AP /N. + The base case: PDFium's text API cannot see it + until the page is flattened. + 2. `invoice_date` value painted through a *nested* form XObject + (`/Tx BMC q /Fm0 Do Q EMC`), the shape Acrobat and + several server-side fillers emit. Detecting it + requires descending into form objects rather than + only inspecting the appearance's top-level objects. + 3. `amount` value painted by the /AP *and* drawn into the page + content stream at the same spot, as partially + flattened files do. Must be extracted once, not + twice. Its rect also covers a `PREPRINTED-LABEL` + that only the content stream draws: flattening + replaces the content under a widget rect, so that + label must be restored rather than lost. + 4. `default_only_choice` /V is set but no appearance paints it. An unpainted + default is not visible text and must stay out of + the text layer (it remains available as structured + form metadata). + 5. `hidden_note` /AP paints text but the annotation carries the + Hidden flag, so it is never rendered and must not + reach the text layer. + 6. (freetext annotation) a non-widget annotation whose appearance paints + text. Flattening must not promote it. + +Page 2 — 1 annotation, 1 form field + `complexity_sentinel` a short value ("OK") on an otherwise empty page, + so the page stays under the "almost no text" + complexity threshold after flattening. + +Page 3 — 1 annotation, 1 form field + `nested_only` the nested-XObject case again, but as the only + annotation on the page. Page 1's `invoice_date` + rides along with text-painting neighbours that + would trigger the flatten anyway; here nothing + else does, so the value is recovered only if the + appearance walk descends into form XObjects. + +Usage: python3 scripts/generate_filled_acroform_fixture.py +""" + +import pathlib + +OUT = ( + pathlib.Path(__file__).resolve().parent.parent + / "integration_tests_data" + / "filled_acroform.pdf" +) + +PAGE_W, PAGE_H = 612, 792 + +# Annotation flag bits (PDF 32000-1 table 165). +F_PRINT = 4 +F_HIDDEN = 2 + + +class Pdf: + """Minimal PDF writer: objects are 1-indexed in insertion order.""" + + def __init__(self): + self.objects = [None] # index 0 unused so object numbers start at 1 + + def reserve(self): + self.objects.append(None) + return len(self.objects) - 1 + + def put(self, num, body): + self.objects[num] = body + return num + + def add(self, body): + return self.put(self.reserve(), body) + + def stream(self, dict_body, content): + data = content.encode("latin-1") + return self.add( + f"<< {dict_body} /Length {len(data)} >>\nstream\n".encode("latin-1") + + data + + b"\nendstream" + ) + + def build(self): + out = bytearray(b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n") + offsets = [0] * len(self.objects) + for num in range(1, len(self.objects)): + body = self.objects[num] + assert body is not None, f"object {num} was reserved but never filled" + if isinstance(body, str): + body = body.encode("latin-1") + offsets[num] = len(out) + out += f"{num} 0 obj\n".encode("latin-1") + body + b"\nendobj\n" + + xref_at = len(out) + count = len(self.objects) + out += f"xref\n0 {count}\n".encode("latin-1") + out += b"0000000000 65535 f \n" + for num in range(1, count): + out += f"{offsets[num]:010d} 00000 n \n".encode("latin-1") + out += ( + f"trailer\n<< /Size {count} /Root 1 0 R >>\nstartxref\n{xref_at}\n".encode( + "latin-1" + ) + + b"%%EOF\n" + ) + return bytes(out) + + +def text_ops(text, font_res, size, x, y): + return f"q BT /{font_res} {size} Tf 0 g {x} {y} Td ({text}) Tj ET Q" + + +def main(): + pdf = Pdf() + + catalog = pdf.reserve() + pages = pdf.reserve() + page1 = pdf.reserve() + page2 = pdf.reserve() + page3 = pdf.reserve() + + helv = pdf.add( + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>" + ) + font_res = f"<< /Font << /Helv {helv} 0 R >> >>" + + def widget(name, value, rect, ap_stream, flags=F_PRINT, page=None, extra=""): + left, bottom, right, top = rect + return pdf.add( + f"<< /Type /Annot /Subtype /Widget /FT /Tx /T ({name}) /V ({value}) " + f"/Rect [{left} {bottom} {right} {top}] /F {flags} /P {page or page1} 0 R " + f"/DA (/Helv 10 Tf 0 g) /AP << /N {ap_stream} 0 R >> {extra}>>" + ) + + def ap(width, height, content, resources=None): + return pdf.stream( + f"/Type /XObject /Subtype /Form /BBox [0 0 {width} {height}] " + f"/Resources {resources or font_res}", + content, + ) + + # 1. Value painted directly by the appearance stream. + customer = widget( + "customer_name", + "ACROFORM-CUSTOMER-7319", + (72, 700, 300, 720), + ap(228, 20, f"/Tx BMC {text_ops('ACROFORM-CUSTOMER-7319', 'Helv', 10, 2, 6)} EMC"), + ) + + # 2. Value painted through a nested form XObject. + inner = ap(228, 20, text_ops("2026-07-28", "Helv", 10, 2, 6)) + date = widget( + "invoice_date", + "2026-07-28", + (72, 660, 300, 680), + ap( + 228, + 20, + f"/Tx BMC q /Fm0 Do Q EMC", + resources=f"<< /XObject << /Fm0 {inner} 0 R >> >>", + ), + ) + + # 3. Value in the appearance *and* in the page content stream. + amount = widget( + "amount", + "50.00", + (72, 620, 300, 640), + ap(228, 20, f"/Tx BMC {text_ops('50.00', 'Helv', 10, 2, 6)} EMC"), + ) + + # 4. Value set, but the appearance paints only a border. + default_only = widget( + "default_only_choice", + "DEFAULT-ONLY-SHOULD-NOT-APPEAR", + (72, 580, 300, 600), + ap(228, 20, "q 0.5 w 0 0 228 20 re S Q"), + ) + + # 5. Appearance paints text, but the annotation is hidden. + hidden = widget( + "hidden_note", + "HIDDEN-SHOULD-NOT-APPEAR", + (72, 540, 300, 560), + ap(228, 20, f"/Tx BMC {text_ops('HIDDEN-SHOULD-NOT-APPEAR', 'Helv', 10, 2, 6)} EMC"), + flags=F_HIDDEN, + ) + + # 6. Non-widget annotation that paints text through its appearance. + freetext = pdf.add( + f"<< /Type /Annot /Subtype /FreeText /Rect [72 500 300 520] /F {F_PRINT} " + f"/P {page1} 0 R /Contents (ANNOTATION-ONLY-SHOULD-NOT-APPEAR) " + f"/DA (/Helv 10 Tf 0 g) /AP << /N " + f"{ap(228, 20, text_ops('ANNOTATION-ONLY-SHOULD-NOT-APPEAR', 'Helv', 10, 2, 6))} 0 R >> >>" + ) + + # Page 3 carries a nested-XObject widget and nothing else, so the page is + # only flattened if the appearance walk descends into form XObjects. On + # page 1 the equivalent widget rides along with its text-painting + # neighbours and would be flattened either way. + nested_inner = ap(228, 20, text_ops("NESTED-ONLY-VALUE", "Helv", 10, 2, 6)) + nested_only = widget( + "nested_only", + "NESTED-ONLY-VALUE", + (72, 700, 300, 720), + ap( + 228, + 20, + "/Tx BMC q /Fm0 Do Q EMC", + resources=f"<< /XObject << /Fm0 {nested_inner} 0 R >> >>", + ), + page=page3, + ) + + sentinel_ap = ap(228, 20, f"/Tx BMC {text_ops('OK', 'Helv', 10, 2, 6)} EMC") + sentinel = pdf.add( + f"<< /Type /Annot /Subtype /Widget /FT /Tx /T (complexity_sentinel) /V (OK) " + f"/Rect [72 700 300 720] /F {F_PRINT} /P {page2} 0 R " + f"/DA (/Helv 10 Tf 0 g) /AP << /N {sentinel_ap} 0 R >> >>" + ) + + # Page 1 content, all of it drawn *before* any widget appearance: + # - a plain title, well clear of every widget rect; + # - the `amount` value, at the exact origin its appearance paints it too; + # - a pre-printed label at the exact origin of the `customer_name` + # appearance, which no appearance reproduces. + # + # PDFium's text layer suppresses one of two runs that start at essentially + # the same point, so both of these collide with a flattened appearance. The + # first collision is between identical strings and is exactly the dedup a + # partially flattened file needs — the value must come out once. The second + # is between different strings, where suppression is pure data loss, so the + # label has to be restored. + page1_content = pdf.stream( + "", + "\n".join( + [ + text_ops("Invoice", "Helv", 14, 72, 750), + text_ops("50.00", "Helv", 10, 74, 626), + text_ops("PREPRINTED-LABEL", "Helv", 10, 74, 706), + ] + ), + ) + # Pages 2 and 3 stay empty so each page's widget is the only text on it. + page2_content = pdf.stream("", "") + page3_content = pdf.stream("", "") + + page1_annots = [customer, date, amount, default_only, hidden, freetext] + pdf.put( + page1, + f"<< /Type /Page /Parent {pages} 0 R /MediaBox [0 0 {PAGE_W} {PAGE_H}] " + f"/Resources {font_res} /Contents {page1_content} 0 R " + f"/Annots [{' '.join(f'{n} 0 R' for n in page1_annots)}] >>", + ) + pdf.put( + page2, + f"<< /Type /Page /Parent {pages} 0 R /MediaBox [0 0 {PAGE_W} {PAGE_H}] " + f"/Resources {font_res} /Contents {page2_content} 0 R " + f"/Annots [{sentinel} 0 R] >>", + ) + pdf.put( + page3, + f"<< /Type /Page /Parent {pages} 0 R /MediaBox [0 0 {PAGE_W} {PAGE_H}] " + f"/Resources {font_res} /Contents {page3_content} 0 R " + f"/Annots [{nested_only} 0 R] >>", + ) + pdf.put( + pages, + f"<< /Type /Pages /Kids [{page1} 0 R {page2} 0 R {page3} 0 R] /Count 3 >>", + ) + + fields = page1_annots[:5] + [sentinel, nested_only] + pdf.put( + catalog, + f"<< /Type /Catalog /Pages {pages} 0 R /AcroForm << " + f"/Fields [{' '.join(f'{n} 0 R' for n in fields)}] " + f"/DA (/Helv 10 Tf 0 g) /DR {font_res} >> >>", + ) + + OUT.write_bytes(pdf.build()) + print(f"wrote {OUT} ({OUT.stat().st_size} bytes)") + + +if __name__ == "__main__": + main()