From 975790517a005f8a0fe6e63dd4fac72ce4864f08 Mon Sep 17 00:00:00 2001 From: Pierre-Loic doulcet Date: Mon, 27 Jul 2026 16:52:52 +0800 Subject: [PATCH 01/50] feat: expose PDF document provenance metadata --- README.md | 8 +- crates/liteparse-napi/src/types.rs | 41 ++++ crates/liteparse-python/src/lib.rs | 55 ++++++ crates/liteparse-wasm/src/lib.rs | 55 ++++++ crates/liteparse/src/document_metadata.rs | 219 +++++++++++++++++++++ crates/liteparse/src/lib.rs | 3 +- crates/liteparse/src/output/json.rs | 2 + crates/liteparse/src/parser.rs | 20 +- crates/liteparse/src/types.rs | 34 ++++ crates/liteparse/tests/integration_test.rs | 22 ++- crates/pdfium-sys/bindings.rs | 16 ++ crates/pdfium-sys/src/dynamic.rs | 18 ++ crates/pdfium-sys/wrapper.h | 1 + crates/pdfium/src/document.rs | 74 +++++++ crates/pdfium/src/lib.rs | 2 +- packages/node/README.md | 6 +- packages/node/native.d.ts | 17 ++ packages/node/src/lib.ts | 24 +++ packages/node/src/native.ts | 17 ++ packages/python/README.md | 6 + packages/python/liteparse/__init__.py | 2 + packages/python/liteparse/parser.py | 23 +++ packages/python/liteparse/types.py | 22 +++ 23 files changed, 678 insertions(+), 9 deletions(-) create mode 100644 crates/liteparse/src/document_metadata.rs diff --git a/README.md b/README.md index f11568f4..57480cb8 100644 --- a/README.md +++ b/README.md @@ -256,8 +256,12 @@ link annotations. The field is absent by default; enabled untagged pages contain ### Document metadata, content bounds, and XFA packets Parse results (Rust/Node/Python APIs) carry the document's `/Info` `creator` -and `producer` entries when present; these are API-only and never appear in -CLI JSON output. Enable `--extract-content-bounds` (Rust/Python +and `producer` entries when present, plus `doc_meta` +(`docMeta` in JavaScript/WASM) with the `/Info` creation/modification +dates, PDF version and encryption permissions, signature state, incremental-save +markers, trailer ID comparison, raw XMP packet (capped at 64 KiB), and source +file size. These document fields are API-only and never appear in CLI JSON +output. Enable `--extract-content-bounds` (Rust/Python `extract_content_bounds`, JavaScript/WASM `extractContentBounds`) to add a per-page `content_bounds`: the union bbox of the page's top-level content objects in viewport coords (absent for empty pages). Enable diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs index 99b90021..dd8e07c8 100644 --- a/crates/liteparse-napi/src/types.rs +++ b/crates/liteparse-napi/src/types.rs @@ -874,10 +874,50 @@ pub struct JsParseResult { pub creator: Option, /// The document's `/Info` `Producer` entry, when present. pub producer: Option, + /// Document-level provenance metadata. + pub doc_meta: JsDocumentMetadata, /// Raw XFA packets; present only when `extractXfaPackets` is enabled. pub xfa_packets: Option>, } +#[napi(object)] +#[derive(Clone)] +pub struct JsDocumentMetadata { + pub creation_date: Option, + pub mod_date: Option, + pub file_version: Option, + pub is_encrypted: Option, + pub security_handler_revision: Option, + pub permissions: Option, + pub eof_section_count: Option, + pub startxref_count: Option, + pub trailer_id_pair_differs: Option, + pub raw_file_size: Option, + pub xmp: Option, + pub signature_count: Option, + pub signature_byte_range_reaches_eof: Option, +} + +impl JsDocumentMetadata { + fn from_rust(metadata: &liteparse::types::DocumentMetadata) -> Self { + Self { + creation_date: metadata.creation_date.clone(), + mod_date: metadata.mod_date.clone(), + file_version: metadata.file_version, + is_encrypted: metadata.is_encrypted, + security_handler_revision: metadata.security_handler_revision, + permissions: metadata.permissions.map(|value| value as f64), + eof_section_count: metadata.eof_section_count, + startxref_count: metadata.startxref_count, + trailer_id_pair_differs: metadata.trailer_id_pair_differs, + raw_file_size: metadata.raw_file_size.map(|value| value as f64), + xmp: metadata.xmp.clone(), + signature_count: metadata.signature_count, + signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, + } + } +} + /// One raw packet from an XFA form document's `/XFA` array. #[napi(object)] #[derive(Clone)] @@ -1063,6 +1103,7 @@ impl JsParseResult { form_type: result.form_type, creator: result.creator.clone(), producer: result.producer.clone(), + doc_meta: JsDocumentMetadata::from_rust(&result.doc_meta), xfa_packets: result .xfa_packets .as_ref() diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs index 9cf5839a..851dce7d 100644 --- a/crates/liteparse-python/src/lib.rs +++ b/crates/liteparse-python/src/lib.rs @@ -610,9 +610,62 @@ struct PyParseResult { #[pyo3(get)] producer: Option, #[pyo3(get)] + doc_meta: PyDocumentMetadata, + #[pyo3(get)] xfa_packets: Option>, } +#[pyclass(frozen, from_py_object)] +#[derive(Clone)] +struct PyDocumentMetadata { + #[pyo3(get)] + creation_date: Option, + #[pyo3(get)] + mod_date: Option, + #[pyo3(get)] + file_version: Option, + #[pyo3(get)] + is_encrypted: Option, + #[pyo3(get)] + security_handler_revision: Option, + #[pyo3(get)] + permissions: Option, + #[pyo3(get)] + eof_section_count: Option, + #[pyo3(get)] + startxref_count: Option, + #[pyo3(get)] + trailer_id_pair_differs: Option, + #[pyo3(get)] + raw_file_size: Option, + #[pyo3(get)] + xmp: Option, + #[pyo3(get)] + signature_count: Option, + #[pyo3(get)] + signature_byte_range_reaches_eof: Option, +} + +impl From for PyDocumentMetadata { + fn from(metadata: liteparse::types::DocumentMetadata) -> Self { + Self { + creation_date: metadata.creation_date, + mod_date: metadata.mod_date, + file_version: metadata.file_version, + is_encrypted: metadata.is_encrypted, + security_handler_revision: metadata.security_handler_revision, + permissions: metadata.permissions, + eof_section_count: metadata.eof_section_count, + startxref_count: metadata.startxref_count, + trailer_id_pair_differs: metadata.trailer_id_pair_differs, + raw_file_size: metadata.raw_file_size, + xmp: metadata.xmp, + signature_count: metadata.signature_count, + signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, + } + } +} + /// One raw packet from an XFA form document's `/XFA` array. #[pyclass(frozen, from_py_object)] #[derive(Clone)] @@ -676,6 +729,7 @@ impl PyParseResult { form_type: result.form_type, creator: result.creator, producer: result.producer, + doc_meta: result.doc_meta.into(), xfa_packets: result.xfa_packets.map(|packets| { packets .into_iter() @@ -1470,6 +1524,7 @@ fn _liteparse(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs index 482690a4..4409e844 100644 --- a/crates/liteparse-wasm/src/lib.rs +++ b/crates/liteparse-wasm/src/lib.rs @@ -612,11 +612,65 @@ pub struct ParseResult { /// The document's `/Info` `Producer` entry, when present. #[serde(skip_serializing_if = "Option::is_none")] pub producer: Option, + /// Document-level provenance metadata. + pub doc_meta: DocumentMetadata, /// Raw XFA packets; present only when `extractXfaPackets` is enabled. #[serde(skip_serializing_if = "Option::is_none")] pub xfa_packets: Option>, } +#[derive(Serialize, Tsify)] +#[tsify(into_wasm_abi)] +#[serde(rename_all = "camelCase")] +pub struct DocumentMetadata { + #[serde(skip_serializing_if = "Option::is_none")] + pub creation_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mod_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_encrypted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub security_handler_revision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub permissions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub eof_section_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub startxref_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub trailer_id_pair_differs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_file_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub xmp: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature_byte_range_reaches_eof: Option, +} + +impl From<&liteparse::types::DocumentMetadata> for DocumentMetadata { + fn from(metadata: &liteparse::types::DocumentMetadata) -> Self { + Self { + creation_date: metadata.creation_date.clone(), + mod_date: metadata.mod_date.clone(), + file_version: metadata.file_version, + is_encrypted: metadata.is_encrypted, + security_handler_revision: metadata.security_handler_revision, + permissions: metadata.permissions.map(|value| value as f64), + eof_section_count: metadata.eof_section_count, + startxref_count: metadata.startxref_count, + trailer_id_pair_differs: metadata.trailer_id_pair_differs, + raw_file_size: metadata.raw_file_size.map(|value| value as f64), + xmp: metadata.xmp.clone(), + signature_count: metadata.signature_count, + signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, + } + } +} + /// One raw packet from an XFA form document's `/XFA` array. #[derive(Serialize, Tsify)] #[tsify(into_wasm_abi)] @@ -980,6 +1034,7 @@ impl LiteParse { form_type: result.form_type, creator: result.creator.clone(), producer: result.producer.clone(), + doc_meta: DocumentMetadata::from(&result.doc_meta), xfa_packets: result.xfa_packets.as_ref().map(|packets| { packets .iter() diff --git a/crates/liteparse/src/document_metadata.rs b/crates/liteparse/src/document_metadata.rs new file mode 100644 index 00000000..05f892da --- /dev/null +++ b/crates/liteparse/src/document_metadata.rs @@ -0,0 +1,219 @@ +//! Document-level PDF provenance metadata extraction. + +use crate::types::{DocumentMetadata, PdfInput}; +use pdfium::Document; +use std::io::{Read, Seek, SeekFrom}; + +const SCAN_BUFFER_BYTES: usize = 1 << 20; +const SCAN_OVERLAP: usize = 64; +const XMP_MAX_BYTES: usize = 64 * 1024; + +pub(crate) fn extract(input: &PdfInput, document: &Document<'_>) -> DocumentMetadata { + let mut metadata = match input { + #[cfg(not(target_arch = "wasm32"))] + PdfInput::Path(path) => std::fs::File::open(path) + .ok() + .map(|mut file| extract_raw_facts(&mut file)) + .unwrap_or_default(), + PdfInput::Bytes(bytes) => extract_raw_facts(&mut std::io::Cursor::new(bytes)), + #[cfg(target_arch = "wasm32")] + PdfInput::Path(_) => DocumentMetadata::default(), + }; + + metadata.creation_date = document.meta_text("CreationDate"); + metadata.mod_date = document.meta_text("ModDate"); + metadata.file_version = document.file_version(); + let security_revision = document.security_handler_revision(); + metadata.is_encrypted = Some(security_revision != -1); + if security_revision != -1 { + metadata.security_handler_revision = Some(security_revision); + metadata.permissions = Some(document.permissions()); + } + let signatures = document.signature_summary(metadata.raw_file_size); + metadata.signature_count = Some(signatures.count); + metadata.signature_byte_range_reaches_eof = signatures.byte_range_reaches_eof; + metadata +} + +fn extract_raw_facts(reader: &mut R) -> DocumentMetadata { + let mut metadata = DocumentMetadata::default(); + let file_size = reader.seek(SeekFrom::End(0)).ok(); + metadata.raw_file_size = file_size; + if reader.seek(SeekFrom::Start(0)).is_err() { + return metadata; + } + + let mut buffer = vec![0u8; SCAN_BUFFER_BYTES]; + let mut carry = 0usize; + let mut file_offset = 0u64; + let mut eof_count = 0u32; + let mut startxref_count = 0u32; + let mut xmp_start = None; + + loop { + let got = match reader.read(&mut buffer[carry..]) { + Ok(got) => got, + Err(_) => break, + }; + let window_len = carry + got; + if window_len == 0 { + break; + } + let countable = if got > 0 && window_len > SCAN_OVERLAP { + window_len - SCAN_OVERLAP + } else { + window_len + }; + let window = &buffer[..window_len]; + eof_count = eof_count.saturating_add(count_occurrences_before(window, b"%%EOF", countable)); + startxref_count = startxref_count.saturating_add(count_occurrences_before( + window, + b"startxref", + countable, + )); + if xmp_start.is_none() + && let Some(offset) = find_bytes(window, b"") + .map(|offset| packet_end + offset + 2) + .unwrap_or(packet_end + b" Option { + (!needle.is_empty() && haystack.len() >= needle.len()) + .then(|| { + haystack + .windows(needle.len()) + .position(|part| part == needle) + }) + .flatten() +} + +fn count_occurrences_before(haystack: &[u8], needle: &[u8], start_limit: usize) -> u32 { + let mut count = 0u32; + let mut cursor = 0usize; + while let Some(offset) = find_bytes(&haystack[cursor..], needle) { + if cursor + offset >= start_limit { + break; + } + count = count.saturating_add(1); + cursor += offset + needle.len(); + } + count +} + +fn trailer_id_pair_differs(bytes: &[u8]) -> Option { + let mut cursor = 0usize; + let mut last_pair: Option<(&[u8], &[u8])> = None; + while let Some(offset) = find_bytes(&bytes[cursor..], b"/ID") { + let id_start = cursor + offset; + let mut pos = id_start + 3; + while bytes + .get(pos) + .is_some_and(|b| matches!(b, b' ' | b'\r' | b'\n' | b'\t')) + { + pos += 1; + } + if bytes.get(pos) == Some(&b'[') { + pos += 1; + let mut values = Vec::with_capacity(2); + while pos < bytes.len() && bytes[pos] != b']' && values.len() < 2 { + if bytes[pos] == b'<' { + let start = pos + 1; + if let Some(close) = bytes[start..].iter().position(|b| *b == b'>') { + values.push(&bytes[start..start + close]); + pos = start + close; + } + } + pos += 1; + } + if values.len() == 2 { + last_pair = Some((values[0], values[1])); + } + } + cursor = id_start + 3; + } + last_pair.map(|(first, second)| first != second) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_raw_provenance_facts() { + let pdf = b"%PDF-1.4\n/ID []\nstartxref\n1\n%%EOF\n\ + update\n/ID [ ]\nstartxref\n2\n%%EOF\n\ + oktail"; + let metadata = extract_raw_facts(&mut std::io::Cursor::new(pdf)); + assert_eq!(metadata.raw_file_size, Some(pdf.len() as u64)); + assert_eq!(metadata.eof_section_count, Some(2)); + assert_eq!(metadata.startxref_count, Some(2)); + assert_eq!(metadata.trailer_id_pair_differs, Some(true)); + assert_eq!( + metadata.xmp.as_deref(), + Some("ok") + ); + } + + #[test] + fn trailer_id_uses_last_valid_pair() { + assert_eq!( + trailer_id_pair_differs(b"/ID [] junk /ID []"), + Some(false) + ); + assert_eq!(trailer_id_pair_differs(b"no trailer id"), None); + } + + #[test] + fn finds_markers_that_cross_scan_chunks_without_double_counting() { + let mut pdf = vec![b'x'; SCAN_BUFFER_BYTES - 7]; + pdf.extend_from_slice(b"startxref\n%%EOF\npayload"); + let metadata = extract_raw_facts(&mut std::io::Cursor::new(pdf)); + assert_eq!(metadata.startxref_count, Some(1)); + assert_eq!(metadata.eof_section_count, Some(1)); + assert_eq!( + metadata.xmp.as_deref(), + Some("payload") + ); + } +} diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rs index 56dd89b9..f4d5582f 100644 --- a/crates/liteparse/src/lib.rs +++ b/crates/liteparse/src/lib.rs @@ -12,7 +12,7 @@ pub use font_db_resolver::FontDbResolver; pub use glyph_resolver::{GLYPH_RESOLVER_FONT_SIZE, GlyphResolver}; pub use parser::{LiteParse, ParseResult, ScreenshotResult}; pub use search::{SearchOptions, search_items}; -pub use types::{ParsedPage, TextItem, WordBox}; +pub use types::{DocumentMetadata, ParsedPage, TextItem, WordBox}; // ── Modules with user-facing types (visible in docs) ─────────────────── pub mod config; @@ -28,6 +28,7 @@ mod acroform_repair; #[cfg(not(target_arch = "wasm32"))] #[doc(hidden)] pub mod conversion; +mod document_metadata; #[doc(hidden)] pub mod extract; #[doc(hidden)] diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rs index 01bd782f..173d9db7 100644 --- a/crates/liteparse/src/output/json.rs +++ b/crates/liteparse/src/output/json.rs @@ -395,6 +395,7 @@ mod tests { form_type: None, creator: Some("LibreOffice".into()), producer: Some("LibreOffice 7.4".into()), + doc_meta: crate::types::DocumentMetadata::default(), xfa_packets: Some(vec![crate::types::XfaPacket { index: 0, name: Some("datasets".into()), @@ -406,6 +407,7 @@ mod tests { serde_json::from_str(&format_json_result(&result, false).unwrap()).unwrap(); assert!(value.get("creator").is_none()); assert!(value.get("producer").is_none()); + assert!(value.get("doc_meta").is_none()); assert_eq!(value["xfa_packets"][0]["name"], "datasets"); assert_eq!(value["xfa_packets"][0]["content_length"], 11); assert_eq!(value["images"][0]["bbox"]["x"], 10.0); diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs index 4f387ac4..43a865b0 100644 --- a/crates/liteparse/src/parser.rs +++ b/crates/liteparse/src/parser.rs @@ -14,7 +14,8 @@ use crate::projection; #[cfg(not(target_arch = "wasm32"))] use crate::render; use crate::types::{ - ExtractedImage, OutlineTarget, Page, ParsedPage, PdfInput, ScreenshotRect, XfaPacket, + DocumentMetadata, ExtractedImage, OutlineTarget, Page, ParsedPage, PdfInput, ScreenshotRect, + XfaPacket, }; use pdfium::Library; @@ -42,6 +43,9 @@ pub struct ParseResult { pub creator: Option, /// The document's `/Info` `Producer` entry, when present. pub producer: Option, + /// Document provenance metadata (dates, version/security, signatures, + /// incremental-save markers, trailer IDs, raw XMP, and source size). + pub doc_meta: DocumentMetadata, /// Raw XFA packets, present only when `extract_xfa_packets` is enabled. /// `Some([])` means extraction ran on a non-XFA document. pub xfa_packets: Option>, @@ -429,6 +433,7 @@ impl LiteParse { form_type, creator, producer, + doc_meta, xfa_packets, ) = { let lib = Library::init(); @@ -455,6 +460,16 @@ impl LiteParse { .then(|| document.form_type()); let creator = document.meta_text("Creator"); let producer = document.meta_text("Producer"); + #[cfg(not(target_arch = "wasm32"))] + let doc_meta = if repaired_input.is_some() { + let source_document = + extract::load_document_from_input(&lib, &validated_input, password)?; + crate::document_metadata::extract(&validated_input, &source_document) + } else { + crate::document_metadata::extract(&validated_input, &document) + }; + #[cfg(target_arch = "wasm32")] + let doc_meta = crate::document_metadata::extract(&validated_input, &document); let xfa_packets = self.config.extract_xfa_packets.then(|| { document .xfa_packets() @@ -540,6 +555,7 @@ impl LiteParse { form_type, creator, producer, + doc_meta, xfa_packets, ) }; @@ -631,6 +647,7 @@ impl LiteParse { form_type, creator, producer, + doc_meta, xfa_packets, }) } @@ -671,6 +688,7 @@ impl LiteParse { form_type: None, creator: None, producer: None, + doc_meta: DocumentMetadata::default(), xfa_packets: None, } } diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs index aacaee4f..01e55f96 100644 --- a/crates/liteparse/src/types.rs +++ b/crates/liteparse/src/types.rs @@ -10,6 +10,40 @@ pub enum PdfInput { Bytes(Vec), } +/// Document-level provenance metadata extracted from PDFium plus a bounded +/// streaming scan of the source PDF. Fields stay optional so malformed +/// metadata never prevents the document itself from being parsed. +#[derive(Debug, Clone, Default, Serialize)] +pub struct DocumentMetadata { + #[serde(skip_serializing_if = "Option::is_none")] + pub creation_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mod_date: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_encrypted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub security_handler_revision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub permissions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub eof_section_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub startxref_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub trailer_id_pair_differs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_file_size: Option, + /// Raw XMP packet text, capped at 64 KiB. + #[serde(skip_serializing_if = "Option::is_none")] + pub xmp: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub signature_byte_range_reaches_eof: Option, +} + /// Represents a single text item extracted from a PDF page, /// including its content, position, size, rotation, and font metadata. #[derive(Debug, Clone, Default, Serialize)] diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs index 186f2bd4..42e5c6cd 100644 --- a/crates/liteparse/tests/integration_test.rs +++ b/crates/liteparse/tests/integration_test.rs @@ -151,28 +151,46 @@ async fn test_parse_office_doc_integration() { #[tokio::test] #[serial] async fn test_parse_pdf_integration() { - let lit = LiteParse::new(LiteParseConfig::default()); + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + ..LiteParseConfig::default() + }); let parsed = lit .parse("../../integration_tests_data/sample.pdf") .await .expect("Should be able to parse"); assert_eq!(parsed.pages.len(), 1); + assert!(parsed.doc_meta.file_version.is_some()); + assert_eq!(parsed.doc_meta.is_encrypted, Some(false)); + assert!(parsed.doc_meta.raw_file_size.is_some_and(|size| size > 0)); + assert!( + parsed + .doc_meta + .eof_section_count + .is_some_and(|count| count > 0) + ); + assert_eq!(parsed.doc_meta.signature_count, Some(0)); } #[tokio::test] #[serial] async fn test_parse_bytes_pdf_integration() { let fixture_path = "../../integration_tests_data/sample.pdf"; - let lit = LiteParse::new(LiteParseConfig::default()); + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + ..LiteParseConfig::default() + }); let data = tokio::fs::read(fixture_path) .await .expect("Should be able to read file"); + let expected_size = data.len() as u64; let input = PdfInput::Bytes(data); let parsed = lit .parse_input(input) .await .expect("Should be able to parse"); assert_eq!(parsed.pages.len(), 1); + assert_eq!(parsed.doc_meta.raw_file_size, Some(expected_size)); } /// Stress test: many concurrent `parse_input` calls on a multi-threaded diff --git a/crates/pdfium-sys/bindings.rs b/crates/pdfium-sys/bindings.rs index 27a04774..ed73e497 100644 --- a/crates/pdfium-sys/bindings.rs +++ b/crates/pdfium-sys/bindings.rs @@ -563,6 +563,22 @@ unsafe extern "C" { fileVersion: *mut ::std::os::raw::c_int, ) -> FPDF_BOOL; } +unsafe extern "C" { + pub fn FPDF_GetSignatureCount(document: FPDF_DOCUMENT) -> ::std::os::raw::c_int; +} +unsafe extern "C" { + pub fn FPDF_GetSignatureObject( + document: FPDF_DOCUMENT, + index: ::std::os::raw::c_int, + ) -> FPDF_SIGNATURE; +} +unsafe extern "C" { + pub fn FPDFSignatureObj_GetByteRange( + signature: FPDF_SIGNATURE, + buffer: *mut ::std::os::raw::c_int, + length: ::std::os::raw::c_ulong, + ) -> ::std::os::raw::c_ulong; +} unsafe extern "C" { pub fn FPDF_GetLastError() -> ::std::os::raw::c_ulong; } diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rs index b1fb7bb4..7a595e8a 100644 --- a/crates/pdfium-sys/src/dynamic.rs +++ b/crates/pdfium-sys/src/dynamic.rs @@ -67,6 +67,18 @@ pub struct PdfiumBindings { *mut std::os::raw::c_void, std::os::raw::c_ulong, ) -> std::os::raw::c_ulong, + pub FPDF_GetFileVersion: + unsafe extern "C" fn(FPDF_DOCUMENT, *mut std::os::raw::c_int) -> FPDF_BOOL, + pub FPDF_GetSecurityHandlerRevision: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_int, + pub FPDF_GetDocPermissions: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_ulong, + pub FPDF_GetSignatureCount: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_int, + pub FPDF_GetSignatureObject: + unsafe extern "C" fn(FPDF_DOCUMENT, std::os::raw::c_int) -> FPDF_SIGNATURE, + pub FPDFSignatureObj_GetByteRange: unsafe extern "C" fn( + FPDF_SIGNATURE, + *mut std::os::raw::c_int, + std::os::raw::c_ulong, + ) -> std::os::raw::c_ulong, pub FPDF_GetXFAPacketCount: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_int, pub FPDF_GetXFAPacketName: unsafe extern "C" fn( FPDF_DOCUMENT, @@ -524,6 +536,12 @@ impl PdfiumBindings { FORM_DoPageAAction: load_fn!(lib, "FORM_DoPageAAction"), FPDF_FFLDraw: load_fn!(lib, "FPDF_FFLDraw"), FPDF_GetMetaText: load_fn!(lib, "FPDF_GetMetaText"), + FPDF_GetFileVersion: load_fn!(lib, "FPDF_GetFileVersion"), + FPDF_GetSecurityHandlerRevision: load_fn!(lib, "FPDF_GetSecurityHandlerRevision"), + FPDF_GetDocPermissions: load_fn!(lib, "FPDF_GetDocPermissions"), + FPDF_GetSignatureCount: load_fn!(lib, "FPDF_GetSignatureCount"), + FPDF_GetSignatureObject: load_fn!(lib, "FPDF_GetSignatureObject"), + FPDFSignatureObj_GetByteRange: load_fn!(lib, "FPDFSignatureObj_GetByteRange"), FPDF_GetXFAPacketCount: load_fn!(lib, "FPDF_GetXFAPacketCount"), FPDF_GetXFAPacketName: load_fn!(lib, "FPDF_GetXFAPacketName"), FPDF_GetXFAPacketContent: load_fn!(lib, "FPDF_GetXFAPacketContent"), diff --git a/crates/pdfium-sys/wrapper.h b/crates/pdfium-sys/wrapper.h index e35d8416..46d184d0 100644 --- a/crates/pdfium-sys/wrapper.h +++ b/crates/pdfium-sys/wrapper.h @@ -5,3 +5,4 @@ #include "fpdf_annot.h" #include "fpdf_structtree.h" #include "fpdf_transformpage.h" +#include "fpdf_signature.h" diff --git a/crates/pdfium/src/document.rs b/crates/pdfium/src/document.rs index 7e7f18ef..19d2ea83 100644 --- a/crates/pdfium/src/document.rs +++ b/crates/pdfium/src/document.rs @@ -50,6 +50,14 @@ pub struct XfaPacket { pub content: Option>, } +/// Signature summary used for document provenance metadata. +#[derive(Debug, Clone, Copy)] +pub struct SignatureSummary { + pub count: u32, + /// `None` when signatures exist but PDFium did not expose any byte range. + pub byte_range_reaches_eof: Option, +} + impl<'lib> Document<'lib> { pub fn page_count(&self) -> i32 { unsafe { ffi!(FPDF_GetPageCount(self.handle)) } @@ -133,6 +141,72 @@ impl<'lib> Document<'lib> { Some(String::from_utf16_lossy(&buf[..end])) } + /// Encoded PDF version (`14` means PDF 1.4), when present. + pub fn file_version(&self) -> Option { + let mut version = 0; + let ok = unsafe { ffi!(FPDF_GetFileVersion(self.handle, &mut version)) }; + (ok != 0).then_some(version) + } + + /// PDF security-handler revision, or `-1` for an unencrypted document. + pub fn security_handler_revision(&self) -> i32 { + unsafe { ffi!(FPDF_GetSecurityHandlerRevision(self.handle)) } + } + + /// Document permission flags reported by PDFium. + pub fn permissions(&self) -> u64 { + unsafe { ffi!(FPDF_GetDocPermissions(self.handle)) as u64 } + } + + /// Count signatures and determine whether every readable final byte-range + /// segment reaches the current end of the file. + pub fn signature_summary(&self, file_size: Option) -> SignatureSummary { + const MAX_BYTE_RANGE_VALUES: usize = 8; + let count = unsafe { ffi!(FPDF_GetSignatureCount(self.handle)) }.max(0) as u32; + if count == 0 { + return SignatureSummary { + count, + byte_range_reaches_eof: None, + }; + } + + let mut known = false; + let mut reaches_eof = true; + for index in 0..count { + let signature = unsafe { ffi!(FPDF_GetSignatureObject(self.handle, index as i32)) }; + if signature.is_null() { + continue; + } + let mut ranges = [0i32; MAX_BYTE_RANGE_VALUES]; + let len = unsafe { + ffi!(FPDFSignatureObj_GetByteRange( + signature, + ranges.as_mut_ptr(), + ranges.len() as std::os::raw::c_ulong, + )) + } as usize; + if !(2..=MAX_BYTE_RANGE_VALUES).contains(&len) { + continue; + } + known = true; + let start = i64::from(ranges[len - 2]); + let length = i64::from(ranges[len - 1]); + if let Some(file_size) = file_size + && (start < 0 + || length < 0 + || u64::try_from(start + length) + .ok() + .is_some_and(|range_end| range_end < file_size)) + { + reaches_eof = false; + } + } + SignatureSummary { + count, + byte_range_reaches_eof: known.then_some(reaches_eof), + } + } + /// Number of packets in the document's `/XFA` array (0 for non-XFA docs). pub fn xfa_packet_count(&self) -> i32 { unsafe { ffi!(FPDF_GetXFAPacketCount(self.handle)) } diff --git a/crates/pdfium/src/lib.rs b/crates/pdfium/src/lib.rs index 48f607da..3ce12e32 100644 --- a/crates/pdfium/src/lib.rs +++ b/crates/pdfium/src/lib.rs @@ -9,7 +9,7 @@ mod text_page; mod types; pub use bitmap::Bitmap; -pub use document::{Document, FormEnvironment, OutlineEntry, XfaPacket}; +pub use document::{Document, FormEnvironment, OutlineEntry, SignatureSummary, XfaPacket}; pub use error::PdfiumError; pub use font::{Font, FontType}; pub use library::Library; diff --git a/packages/node/README.md b/packages/node/README.md index f1972ad1..155e44d7 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -99,8 +99,10 @@ all tagged-PDF roots and recursive elements with type, ID, actual/alternate text title, typed attributes, MCIDs, children, and referenced link annotations. Untagged pages have an empty `roots` array; the field is omitted when disabled. -Every result also carries the document's `/Info` `creator`/`producer` when -present (API-level only, not in CLI JSON), and with `extractContentBounds` +Every result also carries the document's `/Info` `creator`/`producer` and a +`docMeta` provenance object with dates, PDF version/security, signature state, +incremental-save markers, trailer ID comparison, raw XMP (capped at 64 KiB), +and source size. These are API-level only, not in CLI JSON. With `extractContentBounds` each page carries a `contentBounds` union bbox of its top-level content objects. With `extractXfaPackets`, `result.xfaPackets` lists each raw XFA packet (index, name, content length, XML content); non-XFA documents yield an diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts index ad9ad0eb..bc52fc79 100644 --- a/packages/node/native.d.ts +++ b/packages/node/native.d.ts @@ -319,9 +319,26 @@ export interface JsParseResult { creator?: string /** The document's `/Info` `Producer` entry, when present. */ producer?: string + /** Document-level provenance metadata. */ + docMeta: JsDocumentMetadata /** Raw XFA packets; present only when `extractXfaPackets` is enabled. */ xfaPackets?: Array } +export interface JsDocumentMetadata { + creationDate?: string + modDate?: string + fileVersion?: number + isEncrypted?: boolean + securityHandlerRevision?: number + permissions?: number + eofSectionCount?: number + startxrefCount?: number + trailerIdPairDiffers?: boolean + rawFileSize?: number + xmp?: string + signatureCount?: number + signatureByteRangeReachesEof?: boolean +} /** One raw packet from an XFA form document's `/XFA` array. */ export interface JsXfaPacket { index: number diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts index f28e9021..16eb162e 100644 --- a/packages/node/src/lib.ts +++ b/packages/node/src/lib.ts @@ -351,10 +351,32 @@ export interface ParseResult { creator?: string; /** The document's `/Info` `Producer` entry, when present. */ producer?: string; + /** Document-level provenance metadata from PDFium and the source PDF. */ + docMeta: DocumentMetadata; /** Raw XFA packets; present only when `extractXfaPackets` is enabled. */ xfaPackets?: XfaPacket[]; } +/** Provenance and tamper-analysis facts extracted from the source PDF. */ +export interface DocumentMetadata { + creationDate?: string; + modDate?: string; + /** Encoded PDF version (`14` means PDF 1.4). */ + fileVersion?: number; + isEncrypted?: boolean; + securityHandlerRevision?: number; + permissions?: number; + eofSectionCount?: number; + startxrefCount?: number; + trailerIdPairDiffers?: boolean; + rawFileSize?: number; + /** Raw XMP packet text, capped at 64 KiB. */ + xmp?: string; + signatureCount?: number; + /** False when bytes were appended after a readable signature byte range. */ + signatureByteRangeReachesEof?: boolean; +} + /** One raw packet from an XFA form document's `/XFA` array. */ export interface XfaPacket { index: number; @@ -560,6 +582,7 @@ export class LiteParse { formType: result.formType, creator: result.creator, producer: result.producer, + docMeta: result.docMeta, xfaPackets: result.xfaPackets, }; } @@ -584,6 +607,7 @@ export class LiteParse { text: result.text, images: (result.images ?? []).map(toImage), imageErrorCount: result.imageErrorCount ?? 0, + docMeta: result.docMeta, }; } diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts index cb4d06f3..9301a023 100644 --- a/packages/node/src/native.ts +++ b/packages/node/src/native.ts @@ -239,9 +239,26 @@ export interface NativeParseResult { formType?: number; creator?: string; producer?: string; + docMeta: NativeDocumentMetadata; xfaPackets?: NativeXfaPacket[]; } +export interface NativeDocumentMetadata { + creationDate?: string; + modDate?: string; + fileVersion?: number; + isEncrypted?: boolean; + securityHandlerRevision?: number; + permissions?: number; + eofSectionCount?: number; + startxrefCount?: number; + trailerIdPairDiffers?: boolean; + rawFileSize?: number; + xmp?: string; + signatureCount?: number; + signatureByteRangeReachesEof?: boolean; +} + export interface NativeXfaPacket { index: number; name?: string; diff --git a/packages/python/README.md b/packages/python/README.md index c552ad3e..2380507b 100644 --- a/packages/python/README.md +++ b/packages/python/README.md @@ -92,6 +92,12 @@ all tagged-PDF roots and recursive elements with type, ID, actual/alternate text title, typed attributes, MCIDs, children, and referenced link annotations. Untagged pages have an empty ``roots`` list; the field is ``None`` when disabled. +Every result also carries ``creator``/``producer`` from the PDF ``/Info`` +dictionary and a ``doc_meta`` provenance object with dates, PDF +version/security, signature state, incremental-save markers, trailer ID +comparison, raw XMP (capped at 64 KiB), and source size. These document fields +are API-only and do not alter default CLI JSON. + ## Parsing from Bytes Pass raw PDF bytes directly — useful for web uploads or downloaded files: diff --git a/packages/python/liteparse/__init__.py b/packages/python/liteparse/__init__.py index c0697262..ef7adc60 100644 --- a/packages/python/liteparse/__init__.py +++ b/packages/python/liteparse/__init__.py @@ -13,6 +13,7 @@ LiteParseConfig, PageComplexityStats, ParseResult, + DocumentMetadata, XfaPacket, ParsedPage, TextItem, @@ -38,6 +39,7 @@ "StructureTreeElement", "LiteParseConfig", "ParseResult", + "DocumentMetadata", "XfaPacket", "ParsedPage", "TextItem", diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py index 3b78f9a9..b5c41b0b 100644 --- a/packages/python/liteparse/parser.py +++ b/packages/python/liteparse/parser.py @@ -20,6 +20,7 @@ ParsedPage, ParseError, ParseResult, + DocumentMetadata, ScreenshotRect, ScreenshotResult, TextItem, @@ -308,6 +309,7 @@ def _convert_native_result(native_result: Any) -> ParseResult: for img in getattr(native_result, "images", []) ] native_xfa_packets = getattr(native_result, "xfa_packets", None) + native_doc_meta = getattr(native_result, "doc_meta", None) return ParseResult( pages=pages, text=native_result.text, @@ -316,6 +318,27 @@ def _convert_native_result(native_result: Any) -> ParseResult: form_type=getattr(native_result, "form_type", None), creator=getattr(native_result, "creator", None), producer=getattr(native_result, "producer", None), + doc_meta=DocumentMetadata( + creation_date=getattr(native_doc_meta, "creation_date", None), + mod_date=getattr(native_doc_meta, "mod_date", None), + file_version=getattr(native_doc_meta, "file_version", None), + is_encrypted=getattr(native_doc_meta, "is_encrypted", None), + security_handler_revision=getattr( + native_doc_meta, "security_handler_revision", None + ), + permissions=getattr(native_doc_meta, "permissions", None), + eof_section_count=getattr(native_doc_meta, "eof_section_count", None), + startxref_count=getattr(native_doc_meta, "startxref_count", None), + trailer_id_pair_differs=getattr( + native_doc_meta, "trailer_id_pair_differs", None + ), + raw_file_size=getattr(native_doc_meta, "raw_file_size", None), + xmp=getattr(native_doc_meta, "xmp", None), + signature_count=getattr(native_doc_meta, "signature_count", None), + signature_byte_range_reaches_eof=getattr( + native_doc_meta, "signature_byte_range_reaches_eof", None + ), + ), xfa_packets=( [ XfaPacket( diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py index 247e8e19..59eda9e9 100644 --- a/packages/python/liteparse/types.py +++ b/packages/python/liteparse/types.py @@ -214,6 +214,26 @@ class XfaPacket: content: Optional[str] +@dataclass +class DocumentMetadata: + """Document-level provenance metadata from PDFium and the source PDF.""" + creation_date: Optional[str] = None + mod_date: Optional[str] = None + #: Encoded PDF version (14 means PDF 1.4). + file_version: Optional[int] = None + is_encrypted: Optional[bool] = None + security_handler_revision: Optional[int] = None + permissions: Optional[int] = None + eof_section_count: Optional[int] = None + startxref_count: Optional[int] = None + trailer_id_pair_differs: Optional[bool] = None + raw_file_size: Optional[int] = None + #: Raw XMP packet text, capped at 64 KiB. + xmp: Optional[str] = None + signature_count: Optional[int] = None + signature_byte_range_reaches_eof: Optional[bool] = None + + @dataclass class ParseResult: """Result of parsing a document.""" @@ -227,6 +247,8 @@ class ParseResult: creator: Optional[str] = None #: The document's ``/Info`` ``Producer`` entry, when present. producer: Optional[str] = None + #: Document-level provenance metadata. + doc_meta: DocumentMetadata = field(default_factory=DocumentMetadata) #: Raw XFA packets; present only when ``extract_xfa_packets=True``. xfa_packets: Optional[List[XfaPacket]] = None From 24cd52cab457d13b9e7813a7d1ef3fb53cc585ab Mon Sep 17 00:00:00 2001 From: Logan Markewich Date: Tue, 28 Jul 2026 18:58:04 -0600 Subject: [PATCH 02/50] fixes --- Cargo.lock | 1 + README.md | 19 ++- crates/liteparse-napi/src/types.rs | 18 ++- crates/liteparse-python/src/lib.rs | 15 +- crates/liteparse-wasm/src/lib.rs | 20 ++- crates/liteparse/Cargo.toml | 1 + crates/liteparse/src/config.rs | 6 + crates/liteparse/src/conversion.rs | 9 ++ crates/liteparse/src/document_metadata.rs | 164 +++++++++++++-------- crates/liteparse/src/output/json.rs | 2 +- crates/liteparse/src/parser.rs | 33 +++-- crates/liteparse/src/types.rs | 14 +- crates/liteparse/tests/integration_test.rs | 38 +++-- crates/pdfium-sys/src/dynamic.rs | 35 +++-- crates/pdfium/src/document.rs | 76 +++++++--- packages/node/README.md | 11 +- packages/node/native.d.ts | 9 +- packages/node/src/lib.ts | 23 ++- packages/node/src/native.ts | 4 +- packages/python/README.md | 10 +- packages/python/liteparse/parser.py | 30 ++-- packages/python/liteparse/types.py | 13 +- 22 files changed, 396 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cf05ac29..9b65311a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2027,6 +2027,7 @@ dependencies = [ "liteparse-pdfium", "liteparse-pdfium-sys", "lopdf", + "memchr", "oar-ocr", "ordered-float", "regex", diff --git a/README.md b/README.md index 57480cb8..68487dd1 100644 --- a/README.md +++ b/README.md @@ -256,12 +256,19 @@ link annotations. The field is absent by default; enabled untagged pages contain ### Document metadata, content bounds, and XFA packets Parse results (Rust/Node/Python APIs) carry the document's `/Info` `creator` -and `producer` entries when present, plus `doc_meta` -(`docMeta` in JavaScript/WASM) with the `/Info` creation/modification -dates, PDF version and encryption permissions, signature state, incremental-save -markers, trailer ID comparison, raw XMP packet (capped at 64 KiB), and source -file size. These document fields are API-only and never appear in CLI JSON -output. Enable `--extract-content-bounds` (Rust/Python +and `producer` entries when present; these are API-only and never appear in +CLI JSON output. Enable `extract_document_metadata` (JavaScript/WASM +`extractDocumentMetadata`) to add `doc_meta`/`docMeta`, a provenance object +with the `/Info` creation/modification dates, PDF version and encryption +permissions, signature state, incremental-save markers, trailer ID comparison, +the document catalog's XMP packet (capped at 64 KiB, with `xmp_truncated` +when it was cut), and source file size. It is off by default because it +streams the whole source file once; it is absent for inputs converted from a +non-PDF format, where the facts would describe the intermediate PDF rather +than your file. `xmp` needs a structural parse of the document, so it is +skipped (left absent) for sources over 16 MiB and in WASM builds — the other +fields are unaffected. +Enable `--extract-content-bounds` (Rust/Python `extract_content_bounds`, JavaScript/WASM `extractContentBounds`) to add a per-page `content_bounds`: the union bbox of the page's top-level content objects in viewport coords (absent for empty pages). Enable diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs index dd8e07c8..5b0fc8a6 100644 --- a/crates/liteparse-napi/src/types.rs +++ b/crates/liteparse-napi/src/types.rs @@ -65,6 +65,10 @@ pub struct JsLiteParseConfig { /// Extract raw XFA packets (name + XML content) into /// `ParseResult.xfaPackets`. Default false. pub extract_xfa_packets: Option, + /// Collect document provenance metadata into `ParseResult.docMeta`. + /// Default false: it streams the whole source file once. Absent for + /// inputs converted from a non-PDF format. + pub extract_document_metadata: Option, /// Emit each page's `contentBounds` (union bbox of top-level content /// objects, viewport coords). Default false. pub extract_content_bounds: Option, @@ -188,6 +192,9 @@ impl JsLiteParseConfig { if let Some(v) = self.extract_xfa_packets { cfg.extract_xfa_packets = v; } + if let Some(v) = self.extract_document_metadata { + cfg.extract_document_metadata = v; + } if let Some(v) = self.extract_content_bounds { cfg.extract_content_bounds = v; } @@ -264,6 +271,7 @@ impl JsLiteParseConfig { extract_form_fields: Some(cfg.extract_form_fields), extract_structure_tree: Some(cfg.extract_structure_tree), extract_xfa_packets: Some(cfg.extract_xfa_packets), + extract_document_metadata: Some(cfg.extract_document_metadata), extract_content_bounds: Some(cfg.extract_content_bounds), detect_screenshot_rects: Some(cfg.detect_screenshot_rects), render_form_fields: Some(cfg.render_form_fields), @@ -874,8 +882,9 @@ pub struct JsParseResult { pub creator: Option, /// The document's `/Info` `Producer` entry, when present. pub producer: Option, - /// Document-level provenance metadata. - pub doc_meta: JsDocumentMetadata, + /// Document-level provenance metadata; present only when + /// `extractDocumentMetadata` is enabled and the input was a real PDF. + pub doc_meta: Option, /// Raw XFA packets; present only when `extractXfaPackets` is enabled. pub xfa_packets: Option>, } @@ -894,6 +903,8 @@ pub struct JsDocumentMetadata { pub trailer_id_pair_differs: Option, pub raw_file_size: Option, pub xmp: Option, + /// True when the catalog's XMP stream exceeded the 64 KiB cap. + pub xmp_truncated: Option, pub signature_count: Option, pub signature_byte_range_reaches_eof: Option, } @@ -912,6 +923,7 @@ impl JsDocumentMetadata { trailer_id_pair_differs: metadata.trailer_id_pair_differs, raw_file_size: metadata.raw_file_size.map(|value| value as f64), xmp: metadata.xmp.clone(), + xmp_truncated: metadata.xmp_truncated, signature_count: metadata.signature_count, signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, } @@ -1103,7 +1115,7 @@ impl JsParseResult { form_type: result.form_type, creator: result.creator.clone(), producer: result.producer.clone(), - doc_meta: JsDocumentMetadata::from_rust(&result.doc_meta), + doc_meta: result.doc_meta.as_ref().map(JsDocumentMetadata::from_rust), xfa_packets: result .xfa_packets .as_ref() diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs index 851dce7d..9cb6ca61 100644 --- a/crates/liteparse-python/src/lib.rs +++ b/crates/liteparse-python/src/lib.rs @@ -610,7 +610,7 @@ struct PyParseResult { #[pyo3(get)] producer: Option, #[pyo3(get)] - doc_meta: PyDocumentMetadata, + doc_meta: Option, #[pyo3(get)] xfa_packets: Option>, } @@ -641,6 +641,8 @@ struct PyDocumentMetadata { #[pyo3(get)] xmp: Option, #[pyo3(get)] + xmp_truncated: Option, + #[pyo3(get)] signature_count: Option, #[pyo3(get)] signature_byte_range_reaches_eof: Option, @@ -660,6 +662,7 @@ impl From for PyDocumentMetadata { trailer_id_pair_differs: metadata.trailer_id_pair_differs, raw_file_size: metadata.raw_file_size, xmp: metadata.xmp, + xmp_truncated: metadata.xmp_truncated, signature_count: metadata.signature_count, signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, } @@ -729,7 +732,7 @@ impl PyParseResult { form_type: result.form_type, creator: result.creator, producer: result.producer, - doc_meta: result.doc_meta.into(), + doc_meta: result.doc_meta.map(Into::into), xfa_packets: result.xfa_packets.map(|packets| { packets .into_iter() @@ -1052,6 +1055,8 @@ struct PyLiteParseConfig { #[pyo3(get)] extract_xfa_packets: bool, #[pyo3(get)] + extract_document_metadata: bool, + #[pyo3(get)] extract_content_bounds: bool, #[pyo3(get)] detect_screenshot_rects: bool, @@ -1123,6 +1128,7 @@ impl PyLiteParseConfig { extract_form_fields: cfg.extract_form_fields, extract_structure_tree: cfg.extract_structure_tree, extract_xfa_packets: cfg.extract_xfa_packets, + extract_document_metadata: cfg.extract_document_metadata, extract_content_bounds: cfg.extract_content_bounds, detect_screenshot_rects: cfg.detect_screenshot_rects, render_form_fields: cfg.render_form_fields, @@ -1180,6 +1186,7 @@ impl LiteParse { extract_form_fields = None, extract_structure_tree = None, extract_xfa_packets = None, + extract_document_metadata = None, extract_content_bounds = None, detect_screenshot_rects = None, render_form_fields = None, @@ -1214,6 +1221,7 @@ impl LiteParse { extract_form_fields: Option, extract_structure_tree: Option, extract_xfa_packets: Option, + extract_document_metadata: Option, extract_content_bounds: Option, detect_screenshot_rects: Option, render_form_fields: Option, @@ -1298,6 +1306,9 @@ impl LiteParse { if let Some(v) = extract_xfa_packets { cfg.extract_xfa_packets = v; } + if let Some(v) = extract_document_metadata { + cfg.extract_document_metadata = v; + } if let Some(v) = extract_content_bounds { cfg.extract_content_bounds = v; } diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs index 4409e844..f6276efc 100644 --- a/crates/liteparse-wasm/src/lib.rs +++ b/crates/liteparse-wasm/src/lib.rs @@ -62,6 +62,9 @@ pub struct LiteParseConfig { /// Extract raw XFA packets (name + XML content) into /// `ParseResult.xfaPackets`. Default false. extract_xfa_packets: Option, + /// Collect document provenance metadata into `ParseResult.docMeta`. + /// Default false: it streams the whole source file once. + extract_document_metadata: Option, /// Emit each page's `contentBounds` (union bbox of top-level content /// objects, viewport coords). Default false. extract_content_bounds: Option, @@ -170,6 +173,9 @@ impl LiteParseConfig { if let Some(v) = self.extract_xfa_packets { cfg.extract_xfa_packets = v; } + if let Some(v) = self.extract_document_metadata { + cfg.extract_document_metadata = v; + } if let Some(v) = self.extract_content_bounds { cfg.extract_content_bounds = v; } @@ -248,6 +254,7 @@ impl LiteParseConfig { extract_form_fields: Some(cfg.extract_form_fields), extract_structure_tree: Some(cfg.extract_structure_tree), extract_xfa_packets: Some(cfg.extract_xfa_packets), + extract_document_metadata: Some(cfg.extract_document_metadata), extract_content_bounds: Some(cfg.extract_content_bounds), ocr_failure_fatal: Some(cfg.ocr_failure_fatal), ocr_hedge_delays_ms: Some(cfg.ocr_hedge_delays_ms.clone()), @@ -612,8 +619,10 @@ pub struct ParseResult { /// The document's `/Info` `Producer` entry, when present. #[serde(skip_serializing_if = "Option::is_none")] pub producer: Option, - /// Document-level provenance metadata. - pub doc_meta: DocumentMetadata, + /// Document-level provenance metadata; present only when + /// `extractDocumentMetadata` is enabled and the input was a real PDF. + #[serde(skip_serializing_if = "Option::is_none")] + pub doc_meta: Option, /// Raw XFA packets; present only when `extractXfaPackets` is enabled. #[serde(skip_serializing_if = "Option::is_none")] pub xfa_packets: Option>, @@ -645,6 +654,10 @@ pub struct DocumentMetadata { pub raw_file_size: Option, #[serde(skip_serializing_if = "Option::is_none")] pub xmp: Option, + /// True when the catalog's XMP stream exceeded the 64 KiB cap. WASM + /// builds never populate `xmp`, so this is always absent there. + #[serde(skip_serializing_if = "Option::is_none")] + pub xmp_truncated: Option, #[serde(skip_serializing_if = "Option::is_none")] pub signature_count: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -665,6 +678,7 @@ impl From<&liteparse::types::DocumentMetadata> for DocumentMetadata { trailer_id_pair_differs: metadata.trailer_id_pair_differs, raw_file_size: metadata.raw_file_size.map(|value| value as f64), xmp: metadata.xmp.clone(), + xmp_truncated: metadata.xmp_truncated, signature_count: metadata.signature_count, signature_byte_range_reaches_eof: metadata.signature_byte_range_reaches_eof, } @@ -1034,7 +1048,7 @@ impl LiteParse { form_type: result.form_type, creator: result.creator.clone(), producer: result.producer.clone(), - doc_meta: DocumentMetadata::from(&result.doc_meta), + doc_meta: result.doc_meta.as_ref().map(DocumentMetadata::from), xfa_packets: result.xfa_packets.as_ref().map(|packets| { packets .iter() diff --git a/crates/liteparse/Cargo.toml b/crates/liteparse/Cargo.toml index 24eb4930..bdb52859 100644 --- a/crates/liteparse/Cargo.toml +++ b/crates/liteparse/Cargo.toml @@ -31,6 +31,7 @@ blake3 = "1" clap = { version = "4.5.55", features = ["derive"] } file-format = { version = "0.29.0", features = ["reader"] } image = { version = "0.25", default-features = false, features = ["default-formats"] } +memchr = "2" ordered-float = "5.3.0" pdfium = { package = "liteparse-pdfium", version = "1.4.0", path = "../pdfium" } pdfium-sys = { package = "liteparse-pdfium-sys", version = "1.4.0", path = "../pdfium-sys" } diff --git a/crates/liteparse/src/config.rs b/crates/liteparse/src/config.rs index 74e26993..8961e207 100644 --- a/crates/liteparse/src/config.rs +++ b/crates/liteparse/src/config.rs @@ -67,6 +67,11 @@ pub struct LiteParseConfig { /// yield an empty list. #[serde(default)] pub extract_xfa_packets: bool, + /// Collect document-level provenance metadata into `ParseResult.doc_meta`. + /// Default `false`: `None` for inputs converted from a non-PDF format, + /// since the facts would describe the intermediate PDF rather than the caller's file. + #[serde(default)] + pub extract_document_metadata: bool, /// Detect solid rectangles and thick lines in rendered page screenshots /// and attach them to each `ScreenshotResult.rects`. Works on the raster, /// so it also finds structure in scanned/flattened pages that have no @@ -215,6 +220,7 @@ impl Default for LiteParseConfig { extract_structure_tree: false, extract_content_bounds: false, extract_xfa_packets: false, + extract_document_metadata: false, detect_screenshot_rects: false, render_form_fields: false, ocr_failure_fatal: true, diff --git a/crates/liteparse/src/conversion.rs b/crates/liteparse/src/conversion.rs index 8bb407fb..dc45223b 100644 --- a/crates/liteparse/src/conversion.rs +++ b/crates/liteparse/src/conversion.rs @@ -88,6 +88,15 @@ pub struct PdfInputGuard { temps: Vec, } +impl PdfInputGuard { + /// True when the resolved input is a temporary PDF produced by converting + /// a non-PDF source, so raw-file facts describe the intermediate, not the + /// document the caller passed in. + pub fn is_converted(&self) -> bool { + !self.temps.is_empty() + } +} + /// Resolve a document input to a PDF suitable for rendering or text extraction. /// /// When `reject_text_formats` is true, plain-text files (`.txt`, etc.) return a diff --git a/crates/liteparse/src/document_metadata.rs b/crates/liteparse/src/document_metadata.rs index 05f892da..c28ad7b5 100644 --- a/crates/liteparse/src/document_metadata.rs +++ b/crates/liteparse/src/document_metadata.rs @@ -7,6 +7,13 @@ use std::io::{Read, Seek, SeekFrom}; const SCAN_BUFFER_BYTES: usize = 1 << 20; const SCAN_OVERLAP: usize = 64; const XMP_MAX_BYTES: usize = 64 * 1024; +/// Above this size, resolving the catalog's `/Metadata` object costs more than +/// the whole parse: lopdf decodes every stream, so a 103 MB file takes ~7.8 s +/// (worst case measured under the cap is ~0.5 s). Larger documents report no +/// `xmp` at all rather than a cheaper guess — the first `) -> DocumentMetadata { let mut metadata = match input { @@ -30,11 +37,40 @@ pub(crate) fn extract(input: &PdfInput, document: &Document<'_>) -> DocumentMeta metadata.permissions = Some(document.permissions()); } let signatures = document.signature_summary(metadata.raw_file_size); - metadata.signature_count = Some(signatures.count); + metadata.signature_count = signatures.count; metadata.signature_byte_range_reaches_eof = signatures.byte_range_reaches_eof; + + #[cfg(not(target_arch = "wasm32"))] + if let Some((xmp, truncated)) = catalog_xmp(input, metadata.raw_file_size) { + metadata.xmp = Some(xmp); + metadata.xmp_truncated = Some(truncated); + } metadata } +/// Read the document catalog's `/Metadata` XMP stream — the only XMP that is +/// certainly the document's own. `None` when the file is too large to parse +/// cheaply, has no catalog metadata, or cannot be decoded (encrypted or +/// damaged); the caller then reports no XMP. +#[cfg(not(target_arch = "wasm32"))] +fn catalog_xmp(input: &PdfInput, file_size: Option) -> Option<(String, bool)> { + if file_size? > XMP_CATALOG_MAX_FILE_BYTES { + return None; + } + let document = match input { + PdfInput::Path(path) => lopdf::Document::load(path).ok()?, + PdfInput::Bytes(bytes) => lopdf::Document::load_mem(bytes).ok()?, + }; + let object = document.catalog().ok()?.get(b"Metadata").ok()?; + let stream = document.dereference(object).ok()?.1.as_stream().ok()?; + let bytes = stream + .decompressed_content() + .unwrap_or_else(|_| stream.content.clone()); + let truncated = bytes.len() > XMP_MAX_BYTES; + let text = String::from_utf8_lossy(&bytes[..bytes.len().min(XMP_MAX_BYTES)]).into_owned(); + (!text.trim().is_empty()).then_some((text, truncated)) +} + fn extract_raw_facts(reader: &mut R) -> DocumentMetadata { let mut metadata = DocumentMetadata::default(); let file_size = reader.seek(SeekFrom::End(0)).ok(); @@ -45,16 +81,13 @@ fn extract_raw_facts(reader: &mut R) -> DocumentMetadata { let mut buffer = vec![0u8; SCAN_BUFFER_BYTES]; let mut carry = 0usize; - let mut file_offset = 0u64; let mut eof_count = 0u32; let mut startxref_count = 0u32; - let mut xmp_start = None; loop { - let got = match reader.read(&mut buffer[carry..]) { - Ok(got) => got, - Err(_) => break, - }; + // Must be a full fill, not a single `read`: a short read would shrink + // the window below a marker's length and lose it entirely. + let got = fill_buffer(reader, &mut buffer[carry..]); let window_len = carry + got; if window_len == 0 { break; @@ -71,15 +104,9 @@ fn extract_raw_facts(reader: &mut R) -> DocumentMetadata { b"startxref", countable, )); - if xmp_start.is_none() - && let Some(offset) = find_bytes(window, b"(reader: &mut R) -> DocumentMetadata { if let Some(file_size) = file_size { let tail_start = file_size.saturating_sub(SCAN_BUFFER_BYTES as u64); - if reader.seek(SeekFrom::Start(tail_start)).is_ok() { - let mut tail = vec![0u8; (file_size - tail_start) as usize]; - if let Ok(got) = reader.read(&mut tail) { - tail.truncate(got); - metadata.trailer_id_pair_differs = trailer_id_pair_differs(&tail); - } + if reader.seek(SeekFrom::Start(tail_start)).is_ok() + && let Some(tail) = read_up_to(reader, (file_size - tail_start) as usize) + { + metadata.trailer_id_pair_differs = trailer_id_pair_differs(&tail); } } - if let Some(xmp_start) = xmp_start - && reader.seek(SeekFrom::Start(xmp_start)).is_ok() - { - let mut xmp = vec![0u8; XMP_MAX_BYTES]; - if let Ok(got) = reader.read(&mut xmp) { - xmp.truncate(got); - if let Some(packet_end) = find_bytes(&xmp, b"") - .map(|offset| packet_end + offset + 2) - .unwrap_or(packet_end + b"(reader: &mut R, max: usize) -> Option> { + let mut buffer = vec![0u8; max]; + let got = fill_buffer(reader, &mut buffer); + buffer.truncate(got); + (got > 0).then_some(buffer) +} + +/// Fill `buf` completely, looping over short reads. Returns the byte count, +/// short only at EOF or on a read error. +fn fill_buffer(reader: &mut R, buf: &mut [u8]) -> usize { + let mut filled = 0; + while filled < buf.len() { + match reader.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(got) => filled += got, + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(_) => break, } } - - metadata + filled } fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { - (!needle.is_empty() && haystack.len() >= needle.len()) - .then(|| { - haystack - .windows(needle.len()) - .position(|part| part == needle) - }) - .flatten() + memchr::memmem::find(haystack, needle) } fn count_occurrences_before(haystack: &[u8], needle: &[u8], start_limit: usize) -> u32 { - let mut count = 0u32; - let mut cursor = 0usize; - while let Some(offset) = find_bytes(&haystack[cursor..], needle) { - if cursor + offset >= start_limit { - break; - } - count = count.saturating_add(1); - cursor += offset + needle.len(); - } - count + memchr::memmem::find_iter(haystack, needle) + .take_while(|offset| *offset < start_limit) + .count() + .min(u32::MAX as usize) as u32 } fn trailer_id_pair_differs(bytes: &[u8]) -> Option { @@ -182,17 +202,39 @@ mod tests { #[test] fn extracts_raw_provenance_facts() { let pdf = b"%PDF-1.4\n/ID []\nstartxref\n1\n%%EOF\n\ - update\n/ID [ ]\nstartxref\n2\n%%EOF\n\ - oktail"; + update\n/ID [ ]\nstartxref\n2\n%%EOF\ntail"; let metadata = extract_raw_facts(&mut std::io::Cursor::new(pdf)); assert_eq!(metadata.raw_file_size, Some(pdf.len() as u64)); assert_eq!(metadata.eof_section_count, Some(2)); assert_eq!(metadata.startxref_count, Some(2)); assert_eq!(metadata.trailer_id_pair_differs, Some(true)); - assert_eq!( - metadata.xmp.as_deref(), - Some("ok") - ); + // XMP comes from the catalog only; the raw scan never guesses at it. + assert_eq!(metadata.xmp, None); + } + + /// A reader that hands back one byte at a time, like a slow pipe. + struct DripReader(std::io::Cursor>); + + impl Read for DripReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let take = buf.len().min(1); + self.0.read(&mut buf[..take]) + } + } + + impl Seek for DripReader { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.0.seek(pos) + } + } + + #[test] + fn short_reads_do_not_lose_markers_or_the_trailer() { + let pdf = b"%PDF-1.4\nbody\n/ID []\nstartxref\n1\n%%EOF\n"; + let metadata = extract_raw_facts(&mut DripReader(std::io::Cursor::new(pdf.to_vec()))); + assert_eq!(metadata.trailer_id_pair_differs, Some(true)); + assert_eq!(metadata.startxref_count, Some(1)); + assert_eq!(metadata.eof_section_count, Some(1)); } #[test] @@ -207,13 +249,9 @@ mod tests { #[test] fn finds_markers_that_cross_scan_chunks_without_double_counting() { let mut pdf = vec![b'x'; SCAN_BUFFER_BYTES - 7]; - pdf.extend_from_slice(b"startxref\n%%EOF\npayload"); + pdf.extend_from_slice(b"startxref\n%%EOF\npayload"); let metadata = extract_raw_facts(&mut std::io::Cursor::new(pdf)); assert_eq!(metadata.startxref_count, Some(1)); assert_eq!(metadata.eof_section_count, Some(1)); - assert_eq!( - metadata.xmp.as_deref(), - Some("payload") - ); } } diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rs index 173d9db7..bccbf76f 100644 --- a/crates/liteparse/src/output/json.rs +++ b/crates/liteparse/src/output/json.rs @@ -395,7 +395,7 @@ mod tests { form_type: None, creator: Some("LibreOffice".into()), producer: Some("LibreOffice 7.4".into()), - doc_meta: crate::types::DocumentMetadata::default(), + doc_meta: Some(crate::types::DocumentMetadata::default()), xfa_packets: Some(vec![crate::types::XfaPacket { index: 0, name: Some("datasets".into()), diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs index 43a865b0..afd9d10a 100644 --- a/crates/liteparse/src/parser.rs +++ b/crates/liteparse/src/parser.rs @@ -45,7 +45,9 @@ pub struct ParseResult { pub producer: Option, /// Document provenance metadata (dates, version/security, signatures, /// incremental-save markers, trailer IDs, raw XMP, and source size). - pub doc_meta: DocumentMetadata, + /// Present only when `extract_document_metadata` is enabled, and `None` + /// for inputs converted from a non-PDF format. + pub doc_meta: Option, /// Raw XFA packets, present only when `extract_xfa_packets` is enabled. /// `Some([])` means extraction ran on a non-XFA document. pub xfa_packets: Option>, @@ -364,6 +366,13 @@ impl LiteParse { let (validated_input, _guard) = conversion::resolve_pdf_input(input, self.config.password.as_deref(), false).await?; + // Provenance facts describe the file on disk, so they are meaningless + // for a PDF we generated ourselves from a DOCX/XLSX/image. + #[cfg(not(target_arch = "wasm32"))] + let want_doc_meta = self.config.extract_document_metadata && !_guard.is_converted(); + #[cfg(target_arch = "wasm32")] + let want_doc_meta = self.config.extract_document_metadata; + #[cfg(target_arch = "wasm32")] let validated_input = input; @@ -460,16 +469,18 @@ impl LiteParse { .then(|| document.form_type()); let creator = document.meta_text("Creator"); let producer = document.meta_text("Producer"); - #[cfg(not(target_arch = "wasm32"))] - let doc_meta = if repaired_input.is_some() { - let source_document = - extract::load_document_from_input(&lib, &validated_input, password)?; - crate::document_metadata::extract(&validated_input, &source_document) - } else { + let doc_meta = want_doc_meta.then(|| { + // AcroForm repair rewrites the file, so provenance has to come + // from the original document; fall back if it no longer loads. + #[cfg(not(target_arch = "wasm32"))] + if repaired_input.is_some() + && let Ok(source) = + extract::load_document_from_input(&lib, &validated_input, password) + { + return crate::document_metadata::extract(&validated_input, &source); + } crate::document_metadata::extract(&validated_input, &document) - }; - #[cfg(target_arch = "wasm32")] - let doc_meta = crate::document_metadata::extract(&validated_input, &document); + }); let xfa_packets = self.config.extract_xfa_packets.then(|| { document .xfa_packets() @@ -688,7 +699,7 @@ impl LiteParse { form_type: None, creator: None, producer: None, - doc_meta: DocumentMetadata::default(), + doc_meta: None, xfa_packets: None, } } diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs index 01e55f96..b557ab02 100644 --- a/crates/liteparse/src/types.rs +++ b/crates/liteparse/src/types.rs @@ -27,17 +27,29 @@ pub struct DocumentMetadata { pub security_handler_revision: Option, #[serde(skip_serializing_if = "Option::is_none")] pub permissions: Option, + /// Literal `%%EOF` markers in the file. A rough incremental-update signal: + /// embedded PDF attachments and content-stream text inflate it. #[serde(skip_serializing_if = "Option::is_none")] pub eof_section_count: Option, + /// Literal `startxref` markers in the file, with the same caveat. #[serde(skip_serializing_if = "Option::is_none")] pub startxref_count: Option, + /// Whether the two halves of the last trailer `/ID` array differ, which + /// usually means the file was updated after creation. `None` when no + /// hex-string `/ID` pair was found in the last 1 MiB. #[serde(skip_serializing_if = "Option::is_none")] pub trailer_id_pair_differs: Option, #[serde(skip_serializing_if = "Option::is_none")] pub raw_file_size: Option, - /// Raw XMP packet text, capped at 64 KiB. + /// The document catalog's `/Metadata` XMP packet, capped at 64 KiB. + /// `None` when the document has none, when it is too large to resolve + /// cheaply (see `extract_document_metadata`), or in WASM builds. #[serde(skip_serializing_if = "Option::is_none")] pub xmp: Option, + /// True when the catalog's XMP stream exceeded the 64 KiB cap and `xmp` + /// holds only its first 64 KiB. + #[serde(skip_serializing_if = "Option::is_none")] + pub xmp_truncated: Option, #[serde(skip_serializing_if = "Option::is_none")] pub signature_count: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/liteparse/tests/integration_test.rs b/crates/liteparse/tests/integration_test.rs index 42e5c6cd..e08528dd 100644 --- a/crates/liteparse/tests/integration_test.rs +++ b/crates/liteparse/tests/integration_test.rs @@ -153,6 +153,7 @@ async fn test_parse_office_doc_integration() { async fn test_parse_pdf_integration() { let lit = LiteParse::new(LiteParseConfig { ocr_enabled: false, + extract_document_metadata: true, ..LiteParseConfig::default() }); let parsed = lit @@ -160,16 +161,27 @@ async fn test_parse_pdf_integration() { .await .expect("Should be able to parse"); assert_eq!(parsed.pages.len(), 1); - assert!(parsed.doc_meta.file_version.is_some()); - assert_eq!(parsed.doc_meta.is_encrypted, Some(false)); - assert!(parsed.doc_meta.raw_file_size.is_some_and(|size| size > 0)); - assert!( - parsed - .doc_meta - .eof_section_count - .is_some_and(|count| count > 0) - ); - assert_eq!(parsed.doc_meta.signature_count, Some(0)); + let doc_meta = parsed.doc_meta.expect("doc_meta requested"); + assert!(doc_meta.file_version.is_some()); + assert_eq!(doc_meta.is_encrypted, Some(false)); + assert!(doc_meta.raw_file_size.is_some_and(|size| size > 0)); + assert!(doc_meta.eof_section_count.is_some_and(|count| count > 0)); + assert_eq!(doc_meta.signature_count, Some(0)); +} + +/// Provenance is opt-in and stays absent on the default path. +#[tokio::test] +#[serial] +async fn test_doc_meta_absent_unless_requested() { + let lit = LiteParse::new(LiteParseConfig { + ocr_enabled: false, + ..LiteParseConfig::default() + }); + let parsed = lit + .parse("../../integration_tests_data/sample.pdf") + .await + .expect("Should be able to parse"); + assert!(parsed.doc_meta.is_none()); } #[tokio::test] @@ -178,6 +190,7 @@ async fn test_parse_bytes_pdf_integration() { let fixture_path = "../../integration_tests_data/sample.pdf"; let lit = LiteParse::new(LiteParseConfig { ocr_enabled: false, + extract_document_metadata: true, ..LiteParseConfig::default() }); let data = tokio::fs::read(fixture_path) @@ -190,7 +203,10 @@ async fn test_parse_bytes_pdf_integration() { .await .expect("Should be able to parse"); assert_eq!(parsed.pages.len(), 1); - assert_eq!(parsed.doc_meta.raw_file_size, Some(expected_size)); + assert_eq!( + parsed.doc_meta.and_then(|meta| meta.raw_file_size), + Some(expected_size) + ); } /// Stress test: many concurrent `parse_input` calls on a multi-threaded diff --git a/crates/pdfium-sys/src/dynamic.rs b/crates/pdfium-sys/src/dynamic.rs index 7a595e8a..9248c9c6 100644 --- a/crates/pdfium-sys/src/dynamic.rs +++ b/crates/pdfium-sys/src/dynamic.rs @@ -23,6 +23,17 @@ macro_rules! load_fn { }}; } +/// Like `load_fn!`, but yields `None` instead of failing the whole load when +/// the symbol is missing. For APIs a trimmed pdfium build may omit — callers +/// must degrade gracefully rather than assume the function exists. +macro_rules! load_fn_opt { + ($lib:expr, $name:literal) => {{ + unsafe { $lib.get::<*const ()>($name.as_bytes()) } + .ok() + .map(|sym| unsafe { std::mem::transmute(*sym) }) + }}; +} + /// Holds all pdfium function pointers loaded at runtime. pub struct PdfiumBindings { // Keep the library handle alive — dropping it would unload the symbols. @@ -71,14 +82,18 @@ pub struct PdfiumBindings { unsafe extern "C" fn(FPDF_DOCUMENT, *mut std::os::raw::c_int) -> FPDF_BOOL, pub FPDF_GetSecurityHandlerRevision: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_int, pub FPDF_GetDocPermissions: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_ulong, - pub FPDF_GetSignatureCount: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_int, + // `fpdf_signature` is absent from some trimmed pdfium builds, so these + // three load optionally and callers fall back to "no signature info". + pub FPDF_GetSignatureCount: Option std::os::raw::c_int>, pub FPDF_GetSignatureObject: - unsafe extern "C" fn(FPDF_DOCUMENT, std::os::raw::c_int) -> FPDF_SIGNATURE, - pub FPDFSignatureObj_GetByteRange: unsafe extern "C" fn( - FPDF_SIGNATURE, - *mut std::os::raw::c_int, - std::os::raw::c_ulong, - ) -> std::os::raw::c_ulong, + Option FPDF_SIGNATURE>, + pub FPDFSignatureObj_GetByteRange: Option< + unsafe extern "C" fn( + FPDF_SIGNATURE, + *mut std::os::raw::c_int, + std::os::raw::c_ulong, + ) -> std::os::raw::c_ulong, + >, pub FPDF_GetXFAPacketCount: unsafe extern "C" fn(FPDF_DOCUMENT) -> std::os::raw::c_int, pub FPDF_GetXFAPacketName: unsafe extern "C" fn( FPDF_DOCUMENT, @@ -539,9 +554,9 @@ impl PdfiumBindings { FPDF_GetFileVersion: load_fn!(lib, "FPDF_GetFileVersion"), FPDF_GetSecurityHandlerRevision: load_fn!(lib, "FPDF_GetSecurityHandlerRevision"), FPDF_GetDocPermissions: load_fn!(lib, "FPDF_GetDocPermissions"), - FPDF_GetSignatureCount: load_fn!(lib, "FPDF_GetSignatureCount"), - FPDF_GetSignatureObject: load_fn!(lib, "FPDF_GetSignatureObject"), - FPDFSignatureObj_GetByteRange: load_fn!(lib, "FPDFSignatureObj_GetByteRange"), + FPDF_GetSignatureCount: load_fn_opt!(lib, "FPDF_GetSignatureCount"), + FPDF_GetSignatureObject: load_fn_opt!(lib, "FPDF_GetSignatureObject"), + FPDFSignatureObj_GetByteRange: load_fn_opt!(lib, "FPDFSignatureObj_GetByteRange"), FPDF_GetXFAPacketCount: load_fn!(lib, "FPDF_GetXFAPacketCount"), FPDF_GetXFAPacketName: load_fn!(lib, "FPDF_GetXFAPacketName"), FPDF_GetXFAPacketContent: load_fn!(lib, "FPDF_GetXFAPacketContent"), diff --git a/crates/pdfium/src/document.rs b/crates/pdfium/src/document.rs index 19d2ea83..d7c42e07 100644 --- a/crates/pdfium/src/document.rs +++ b/crates/pdfium/src/document.rs @@ -51,13 +51,51 @@ pub struct XfaPacket { } /// Signature summary used for document provenance metadata. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Default)] pub struct SignatureSummary { - pub count: u32, + /// `None` when the loaded pdfium build has no signature API, which is not + /// the same as a document with zero signatures. + pub count: Option, /// `None` when signatures exist but PDFium did not expose any byte range. pub byte_range_reaches_eof: Option, } +/// The `fpdf_signature` entry points, resolved together. `None` when the +/// loaded pdfium build does not export them. +struct SignatureApi { + count: unsafe extern "C" fn(pdfium_sys::FPDF_DOCUMENT) -> std::os::raw::c_int, + object: unsafe extern "C" fn( + pdfium_sys::FPDF_DOCUMENT, + std::os::raw::c_int, + ) -> pdfium_sys::FPDF_SIGNATURE, + byte_range: unsafe extern "C" fn( + pdfium_sys::FPDF_SIGNATURE, + *mut std::os::raw::c_int, + std::os::raw::c_ulong, + ) -> std::os::raw::c_ulong, +} + +impl SignatureApi { + #[cfg(not(target_arch = "wasm32"))] + fn load() -> Option { + let bindings = pdfium_sys::dynamic::pdfium(); + Some(Self { + count: bindings.FPDF_GetSignatureCount?, + object: bindings.FPDF_GetSignatureObject?, + byte_range: bindings.FPDFSignatureObj_GetByteRange?, + }) + } + + #[cfg(target_arch = "wasm32")] + fn load() -> Option { + Some(Self { + count: pdfium_sys::FPDF_GetSignatureCount, + object: pdfium_sys::FPDF_GetSignatureObject, + byte_range: pdfium_sys::FPDFSignatureObj_GetByteRange, + }) + } +} + impl<'lib> Document<'lib> { pub fn page_count(&self) -> i32 { unsafe { ffi!(FPDF_GetPageCount(self.handle)) } @@ -159,31 +197,36 @@ impl<'lib> Document<'lib> { } /// Count signatures and determine whether every readable final byte-range - /// segment reaches the current end of the file. + /// segment reaches the current end of the file. Needs `file_size` to answer + /// the byte-range question at all — without it the verdict stays `None` + /// rather than defaulting to "reaches EOF". pub fn signature_summary(&self, file_size: Option) -> SignatureSummary { const MAX_BYTE_RANGE_VALUES: usize = 8; - let count = unsafe { ffi!(FPDF_GetSignatureCount(self.handle)) }.max(0) as u32; - if count == 0 { + let Some(api) = SignatureApi::load() else { + return SignatureSummary::default(); + }; + let count = unsafe { (api.count)(self.handle) }.max(0) as u32; + let Some(file_size) = file_size.filter(|_| count > 0) else { return SignatureSummary { - count, + count: Some(count), byte_range_reaches_eof: None, }; - } + }; let mut known = false; let mut reaches_eof = true; for index in 0..count { - let signature = unsafe { ffi!(FPDF_GetSignatureObject(self.handle, index as i32)) }; + let signature = unsafe { (api.object)(self.handle, index as i32) }; if signature.is_null() { continue; } let mut ranges = [0i32; MAX_BYTE_RANGE_VALUES]; let len = unsafe { - ffi!(FPDFSignatureObj_GetByteRange( + (api.byte_range)( signature, ranges.as_mut_ptr(), ranges.len() as std::os::raw::c_ulong, - )) + ) } as usize; if !(2..=MAX_BYTE_RANGE_VALUES).contains(&len) { continue; @@ -191,18 +234,17 @@ impl<'lib> Document<'lib> { known = true; let start = i64::from(ranges[len - 2]); let length = i64::from(ranges[len - 1]); - if let Some(file_size) = file_size - && (start < 0 - || length < 0 - || u64::try_from(start + length) - .ok() - .is_some_and(|range_end| range_end < file_size)) + if start < 0 + || length < 0 + || u64::try_from(start + length) + .ok() + .is_some_and(|range_end| range_end < file_size) { reaches_eof = false; } } SignatureSummary { - count, + count: Some(count), byte_range_reaches_eof: known.then_some(reaches_eof), } } diff --git a/packages/node/README.md b/packages/node/README.md index 155e44d7..3666b7fc 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -99,10 +99,13 @@ all tagged-PDF roots and recursive elements with type, ID, actual/alternate text title, typed attributes, MCIDs, children, and referenced link annotations. Untagged pages have an empty `roots` array; the field is omitted when disabled. -Every result also carries the document's `/Info` `creator`/`producer` and a -`docMeta` provenance object with dates, PDF version/security, signature state, -incremental-save markers, trailer ID comparison, raw XMP (capped at 64 KiB), -and source size. These are API-level only, not in CLI JSON. With `extractContentBounds` +Every result also carries the document's `/Info` `creator`/`producer` when +present. With `extractDocumentMetadata`, `result.docMeta` adds a provenance +object with dates, PDF version/security, signature state, incremental-save +markers, trailer ID comparison, the catalog's XMP packet (capped at 64 KiB; +skipped for sources over 16 MiB), and source size — off by default since it +streams the whole file, and absent for inputs converted from a non-PDF +format. These are API-level only, not in CLI JSON. With `extractContentBounds` each page carries a `contentBounds` union bbox of its top-level content objects. With `extractXfaPackets`, `result.xfaPackets` lists each raw XFA packet (index, name, content length, XML content); non-XFA documents yield an diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts index bc52fc79..db77eccb 100644 --- a/packages/node/native.d.ts +++ b/packages/node/native.d.ts @@ -62,6 +62,7 @@ export interface JsLiteParseConfig { * `ParseResult.xfaPackets`. Default false. */ extractXfaPackets?: boolean + extractDocumentMetadata?: boolean /** * Emit each page's `contentBounds` (union bbox of top-level content * objects, viewport coords). Default false. @@ -319,8 +320,11 @@ export interface JsParseResult { creator?: string /** The document's `/Info` `Producer` entry, when present. */ producer?: string - /** Document-level provenance metadata. */ - docMeta: JsDocumentMetadata + /** + * Document-level provenance metadata; present only when + * `extractDocumentMetadata` is enabled and the input was a real PDF. + */ + docMeta?: JsDocumentMetadata /** Raw XFA packets; present only when `extractXfaPackets` is enabled. */ xfaPackets?: Array } @@ -336,6 +340,7 @@ export interface JsDocumentMetadata { trailerIdPairDiffers?: boolean rawFileSize?: number xmp?: string + xmpTruncated?: boolean signatureCount?: number signatureByteRangeReachesEof?: boolean } diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts index 16eb162e..72382db8 100644 --- a/packages/node/src/lib.ts +++ b/packages/node/src/lib.ts @@ -46,6 +46,11 @@ export interface LiteParseConfig { extractStructureTree: boolean; /** Extract raw XFA packets (name + XML content) into `ParseResult.xfaPackets` (default: false). */ extractXfaPackets: boolean; + /** + * Collect document provenance metadata into `result.docMeta`. Default + * false: Absent for inputs converted from a non-PDF format. + */ + extractDocumentMetadata: boolean; /** Emit each page's `contentBounds` (union bbox of top-level content objects) (default: false). */ extractContentBounds: boolean; /** Detect solid rectangles/lines in rendered page screenshots (default: false). */ @@ -351,8 +356,12 @@ export interface ParseResult { creator?: string; /** The document's `/Info` `Producer` entry, when present. */ producer?: string; - /** Document-level provenance metadata from PDFium and the source PDF. */ - docMeta: DocumentMetadata; + /** + * Document-level provenance metadata from PDFium and the source PDF. + * Present only when `extractDocumentMetadata` is enabled and the input was + * a real PDF (not converted from DOCX/XLSX/an image). + */ + docMeta?: DocumentMetadata; /** Raw XFA packets; present only when `extractXfaPackets` is enabled. */ xfaPackets?: XfaPacket[]; } @@ -370,8 +379,14 @@ export interface DocumentMetadata { startxrefCount?: number; trailerIdPairDiffers?: boolean; rawFileSize?: number; - /** Raw XMP packet text, capped at 64 KiB. */ + /** + * The document catalog's `/Metadata` XMP packet, capped at 64 KiB. Absent + * when the document has none, when it is too large to resolve cheaply, or + * in WASM builds. + */ xmp?: string; + /** True when the catalog's XMP stream exceeded the 64 KiB cap. */ + xmpTruncated?: boolean; signatureCount?: number; /** False when bytes were appended after a readable signature byte range. */ signatureByteRangeReachesEof?: boolean; @@ -512,6 +527,7 @@ export class LiteParse { extractFormFields: userConfig.extractFormFields, extractStructureTree: userConfig.extractStructureTree, extractXfaPackets: userConfig.extractXfaPackets, + extractDocumentMetadata: userConfig.extractDocumentMetadata, extractContentBounds: userConfig.extractContentBounds, detectScreenshotRects: userConfig.detectScreenshotRects, renderFormFields: userConfig.renderFormFields, @@ -551,6 +567,7 @@ export class LiteParse { extractFormFields: resolved.extractFormFields ?? false, extractStructureTree: resolved.extractStructureTree ?? false, extractXfaPackets: resolved.extractXfaPackets ?? false, + extractDocumentMetadata: resolved.extractDocumentMetadata ?? false, extractContentBounds: resolved.extractContentBounds ?? false, detectScreenshotRects: resolved.detectScreenshotRects ?? false, renderFormFields: resolved.renderFormFields ?? false, diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts index 9301a023..7194f149 100644 --- a/packages/node/src/native.ts +++ b/packages/node/src/native.ts @@ -37,6 +37,7 @@ export interface LiteParseNativeConfig { extractFormFields?: boolean; extractStructureTree?: boolean; extractXfaPackets?: boolean; + extractDocumentMetadata?: boolean; extractContentBounds?: boolean; detectScreenshotRects?: boolean; renderFormFields?: boolean; @@ -239,7 +240,7 @@ export interface NativeParseResult { formType?: number; creator?: string; producer?: string; - docMeta: NativeDocumentMetadata; + docMeta?: NativeDocumentMetadata; xfaPackets?: NativeXfaPacket[]; } @@ -255,6 +256,7 @@ export interface NativeDocumentMetadata { trailerIdPairDiffers?: boolean; rawFileSize?: number; xmp?: string; + xmpTruncated?: boolean; signatureCount?: number; signatureByteRangeReachesEof?: boolean; } diff --git a/packages/python/README.md b/packages/python/README.md index 2380507b..ea849d69 100644 --- a/packages/python/README.md +++ b/packages/python/README.md @@ -93,10 +93,12 @@ title, typed attributes, MCIDs, children, and referenced link annotations. Untag pages have an empty ``roots`` list; the field is ``None`` when disabled. Every result also carries ``creator``/``producer`` from the PDF ``/Info`` -dictionary and a ``doc_meta`` provenance object with dates, PDF -version/security, signature state, incremental-save markers, trailer ID -comparison, raw XMP (capped at 64 KiB), and source size. These document fields -are API-only and do not alter default CLI JSON. +dictionary. With ``extract_document_metadata=True``, ``result.doc_meta`` adds a +provenance object with dates, PDF version/security, signature state, +incremental-save markers, trailer ID comparison, the catalog's XMP packet +(capped at 64 KiB; skipped for sources over 16 MiB), and source size. It is off by default because it streams the whole source file, +and it is ``None`` for inputs converted from a non-PDF format. These document +fields are API-only and do not alter default CLI JSON. ## Parsing from Bytes diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py index b5c41b0b..56ff32df 100644 --- a/packages/python/liteparse/parser.py +++ b/packages/python/liteparse/parser.py @@ -310,15 +310,10 @@ def _convert_native_result(native_result: Any) -> ParseResult: ] native_xfa_packets = getattr(native_result, "xfa_packets", None) native_doc_meta = getattr(native_result, "doc_meta", None) - return ParseResult( - pages=pages, - text=native_result.text, - images=images, - image_error_count=getattr(native_result, "image_error_count", 0), - form_type=getattr(native_result, "form_type", None), - creator=getattr(native_result, "creator", None), - producer=getattr(native_result, "producer", None), - doc_meta=DocumentMetadata( + doc_meta = ( + None + if native_doc_meta is None + else DocumentMetadata( creation_date=getattr(native_doc_meta, "creation_date", None), mod_date=getattr(native_doc_meta, "mod_date", None), file_version=getattr(native_doc_meta, "file_version", None), @@ -334,11 +329,22 @@ def _convert_native_result(native_result: Any) -> ParseResult: ), raw_file_size=getattr(native_doc_meta, "raw_file_size", None), xmp=getattr(native_doc_meta, "xmp", None), + xmp_truncated=getattr(native_doc_meta, "xmp_truncated", None), signature_count=getattr(native_doc_meta, "signature_count", None), signature_byte_range_reaches_eof=getattr( native_doc_meta, "signature_byte_range_reaches_eof", None ), - ), + ) + ) + return ParseResult( + pages=pages, + text=native_result.text, + images=images, + image_error_count=getattr(native_result, "image_error_count", 0), + form_type=getattr(native_result, "form_type", None), + creator=getattr(native_result, "creator", None), + producer=getattr(native_result, "producer", None), + doc_meta=doc_meta, xfa_packets=( [ XfaPacket( @@ -392,6 +398,7 @@ def __init__( extract_form_fields: Optional[bool] = None, extract_structure_tree: Optional[bool] = None, extract_xfa_packets: Optional[bool] = None, + extract_document_metadata: Optional[bool] = None, extract_content_bounds: Optional[bool] = None, detect_screenshot_rects: Optional[bool] = None, render_form_fields: Optional[bool] = None, @@ -514,6 +521,8 @@ def __init__( kwargs["extract_structure_tree"] = extract_structure_tree if extract_xfa_packets is not None: kwargs["extract_xfa_packets"] = extract_xfa_packets + if extract_document_metadata is not None: + kwargs["extract_document_metadata"] = extract_document_metadata if extract_content_bounds is not None: kwargs["extract_content_bounds"] = extract_content_bounds if detect_screenshot_rects is not None: @@ -696,6 +705,7 @@ def get_config(self) -> LiteParseConfig: extract_images=cfg.extract_images, extract_vector_graphics=cfg.extract_vector_graphics, extract_xfa_packets=cfg.extract_xfa_packets, + extract_document_metadata=cfg.extract_document_metadata, extract_content_bounds=cfg.extract_content_bounds, detect_screenshot_rects=cfg.detect_screenshot_rects, ) diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py index 59eda9e9..4f75d9db 100644 --- a/packages/python/liteparse/types.py +++ b/packages/python/liteparse/types.py @@ -228,8 +228,12 @@ class DocumentMetadata: startxref_count: Optional[int] = None trailer_id_pair_differs: Optional[bool] = None raw_file_size: Optional[int] = None - #: Raw XMP packet text, capped at 64 KiB. + #: The document catalog's ``/Metadata`` XMP packet, capped at 64 KiB. + #: ``None`` when the document has none or it is too large to resolve + #: cheaply. xmp: Optional[str] = None + #: True when the catalog's XMP stream exceeded the 64 KiB cap. + xmp_truncated: Optional[bool] = None signature_count: Optional[int] = None signature_byte_range_reaches_eof: Optional[bool] = None @@ -247,8 +251,10 @@ class ParseResult: creator: Optional[str] = None #: The document's ``/Info`` ``Producer`` entry, when present. producer: Optional[str] = None - #: Document-level provenance metadata. - doc_meta: DocumentMetadata = field(default_factory=DocumentMetadata) + #: Document-level provenance metadata. Present only when + #: ``extract_document_metadata=True`` and the input was a real PDF + #: (not converted from DOCX/XLSX/an image). + doc_meta: Optional[DocumentMetadata] = None #: Raw XFA packets; present only when ``extract_xfa_packets=True``. xfa_packets: Optional[List[XfaPacket]] = None @@ -373,6 +379,7 @@ class LiteParseConfig: extract_images: bool = False extract_vector_graphics: bool = False extract_xfa_packets: bool = False + extract_document_metadata: bool = False detect_screenshot_rects: bool = False extract_content_bounds: bool = False From f7915058855081e61360a37564fd15ec2922e780 Mon Sep 17 00:00:00 2001 From: shuvamk Date: Fri, 31 Jul 2026 00:37:07 +0530 Subject: [PATCH 03/50] fix(markdown): don't backslash-escape inside inline code spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `render_line_inline` escaped `*`, `_` and `\` via `escape_inline()` before handing the text to `apply_style()`, whose mono branch then wrapped the already-escaped text in backticks. CommonMark: "Backslash escapes do not work in code blocks, code spans, autolinks, or raw HTML" — so the inserted backslashes are rendered literally by any conforming renderer. Prose with a monospace run, driven through `parse_from_pages` with `output_format = Markdown`: "Please call the function named" (Helvetica) + "get_user_id" (Courier) + "before you continue with the setup." (Helvetica) before: Please call the function named `get\_user\_id` before ... after: Please call the function named `get_user_id` before ... "Install the runtime under the folder" (Helvetica) + "C:\Program\bin" (Courier) + "and then restart the service daemon." (Helvetica) before: Install the runtime under the folder `C:\\Program\\bin` and ... after: Install the runtime under the folder `C:\Program\bin` and ... Adds `style_body()`, which skips escaping for mono spans, and routes the three escape-then-style call sites through it: the uniform-line fast path and the per-group path in `render_line_inline`, and the uniform path in `render_list_item_text`. The mixed-style fallback at the end of `render_list_item_text` keeps plain `escape_inline` — it has no single style to consult. Only markdown output changes, and only for mono spans that contain `*`, `_` or `\`; the delta is the removal of backslashes that were never meaningful in that position. Co-Authored-By: Claude Opus 5 --- .../liteparse/src/markdown_layout/inline.rs | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/crates/liteparse/src/markdown_layout/inline.rs b/crates/liteparse/src/markdown_layout/inline.rs index e0c1e0d7..a73c566e 100644 --- a/crates/liteparse/src/markdown_layout/inline.rs +++ b/crates/liteparse/src/markdown_layout/inline.rs @@ -44,6 +44,17 @@ pub(super) fn escape_inline(s: &str) -> String { out } +/// Escape `text` for use as the body of a span with `style`. Backslash escapes +/// are inert inside code spans per CommonMark, so mono text is passed through +/// verbatim. +fn style_body(text: &str, style: SpanStyle) -> String { + if style.mono { + text.to_string() + } else { + escape_inline(text) + } +} + /// Wrap `inner` in a markdown inline link to `url`. Uses the angle-bracket /// destination form when the URL contains characters that would otherwise /// terminate or break the `(url)` form (whitespace or parentheses). @@ -122,7 +133,7 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String { if joined.is_empty() { return joined; } - let escaped = escape_inline(&joined); + let escaped = style_body(&joined, styles[0]); if styles[0].is_plain() { return escaped; } @@ -148,7 +159,7 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String { group_text.push_str(span.text.trim()); } let group_text = collapse_whitespace(&group_text); - let escaped = escape_inline(&group_text); + let escaped = style_body(&group_text, style); let mut rendered = if style.is_plain() { escaped } else { @@ -180,7 +191,7 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String { pub(super) fn render_list_item_text(line: &ProjectedLine, marker: &str, rest: &str) -> String { if let Some(style) = line_uniform_style(line) { let plain = collapse_whitespace(rest); - let escaped = escape_inline(&plain); + let escaped = style_body(&plain, style); return if style.is_plain() { escaped } else { @@ -415,4 +426,42 @@ mod tests { assert!(out.contains("call")); assert!(out.contains("on it")); } + + #[test] + fn render_line_inline_mono_span_is_not_escaped() { + let l = styled_line( + &[ + ("call", 50.0, Some("Arial")), + ("get_user_id", 100.0, Some("Courier")), + ("now", 200.0, Some("Arial")), + ], + 100.0, + 10.0, + ); + let out = render_line_inline(&l); + assert!(out.contains("`get_user_id`"), "got: {out}"); + assert!(!out.contains('\\'), "got: {out}"); + } + + #[test] + fn render_line_inline_mono_span_keeps_backslashes() { + let l = styled_line( + &[ + ("open", 50.0, Some("Arial")), + (r"C:\Program\bin", 100.0, Some("Courier")), + ("next", 200.0, Some("Arial")), + ], + 100.0, + 10.0, + ); + let out = render_line_inline(&l); + assert!(out.contains(r"`C:\Program\bin`"), "got: {out}"); + } + + #[test] + fn render_line_inline_uniform_mono_line_is_not_escaped() { + let l = styled_line(&[("a_b*c", 50.0, Some("Courier"))], 100.0, 10.0); + let out = render_line_inline(&l); + assert_eq!(out, "`a_b*c`"); + } } From 3ffc614ed4cb1afe078deee4be877e6889f13514 Mon Sep 17 00:00:00 2001 From: Logan Markewich Date: Fri, 31 Jul 2026 14:43:05 -0600 Subject: [PATCH 04/50] fix: add proper RTL and LTR text detection --- crates/liteparse/src/extract.rs | 89 ++++++++++++++++--- crates/liteparse/src/lib.rs | 1 + .../src/markdown_layout/cross_region.rs | 20 ++++- .../liteparse/src/markdown_layout/inline.rs | 13 ++- .../src/markdown_layout/test_helpers.rs | 3 + crates/liteparse/src/output/markdown.rs | 1 + crates/liteparse/src/projection.rs | 37 ++++++-- crates/liteparse/src/types.rs | 5 ++ 8 files changed, 144 insertions(+), 25 deletions(-) diff --git a/crates/liteparse/src/extract.rs b/crates/liteparse/src/extract.rs index e67d39ce..c6cdf160 100644 --- a/crates/liteparse/src/extract.rs +++ b/crates/liteparse/src/extract.rs @@ -1,3 +1,4 @@ +use crate::bidi::is_rtl_char; use crate::error::LiteParseError; use crate::glyph_names::resolve_glyph_name; use crate::types::{ @@ -1341,22 +1342,26 @@ fn extract_page_text_items( let y_overlap = vp_loose.top < seg.vp_bottom + y_tolerance && vp_loose.bottom > seg.vp_top - y_tolerance; - let gap = vp_strict.left - seg.last_char_right; + // Gaps are measured along the segment's writing direction, so a + // right-to-left run reads exactly like a left-to-right one: positive + // means "further along the line", negative means "doubled back". + let gap = seg.gap_to(&vp_strict, c); // Detect line change using complementary checks: // 1. Strict vertical separation: char's strict top is well below last char's strict bottom - // 2. Line wrap: char goes back leftward AND strict top is below last char's strict bottom - // (even slightly), indicating text wrapped to a new line within the same text object - // 3. Very large leftward jump: if the char jumps back by more than the current + // 2. Line wrap: char goes back against the writing direction AND strict top is below + // last char's strict bottom (even slightly), indicating text wrapped to a new line + // within the same text object + // 3. Very large backward jump: if the char jumps back by more than the current // segment width, it's definitely a new line (handles OCR text with tall bounding // boxes that overlap vertically between lines) let strict_below = vp_strict.top > seg.last_char_bottom; - let large_leftward_jump = gap < -5.0; + let large_backward_jump = gap < -5.0; let seg_width = seg.vp_right - seg.vp_left; - let very_large_leftward_jump = seg_width > 20.0 && gap < -(seg_width * 0.5); + let very_large_backward_jump = seg_width > 20.0 && gap < -(seg_width * 0.5); let line_changed = vp_strict.top > seg.last_char_bottom + y_tolerance - || (strict_below && large_leftward_jump) - || very_large_leftward_jump; + || (strict_below && large_backward_jump) + || very_large_backward_jump; // Dot leader detection: break at the boundary between dots and non-dots. // This prevents items like "Total . . . . 330,100" from merging. @@ -1384,7 +1389,7 @@ fn extract_page_text_items( }; let split = gap >= MAX_INLINE_GAP || (seg.pending_space && gap > seg.avg_char_width() * 2.2); - let loose_gap = vp_strict.left - seg.last_char_loose_right; + let loose_gap = seg.loose_gap_to(&vp_strict, c); let em_vp = (vp_loose.bottom - vp_loose.top).abs(); let space_w = ch.font_space_width().map(|w| w * em_vp).unwrap_or(-1.0); eprintln!( @@ -1429,7 +1434,7 @@ fn extract_page_text_items( .is_some_and(|p| p.is_ascii_alphanumeric()); if prev_alnum && c.is_ascii_alphanumeric() { let em_vp = (vp_loose.bottom - vp_loose.top).abs(); - let loose_gap = vp_strict.left - seg.last_char_loose_right; + let loose_gap = seg.loose_gap_to(&vp_strict, c); if em_vp > 0.0 && loose_gap > 0.0 { let s = font_space_cal.entry(fk.clone()).or_default(); if s.len() < 512 { @@ -1454,7 +1459,7 @@ fn extract_page_text_items( // of the rendered em height as the space estimate. let em_vp = (vp_loose.bottom - vp_loose.top).abs(); let space_w = ch.font_space_width().map(|w| w * em_vp).unwrap_or(0.0); - let loose_gap = vp_strict.left - seg.last_char_loose_right; + let loose_gap = seg.loose_gap_to(&vp_strict, c); let both_alnum = c.is_ascii_alphanumeric() && seg .text @@ -2102,6 +2107,16 @@ struct SegmentBuilder { last_char_right: f32, // Right edge of last char LOOSE bounds (advance-relative gap calculation) last_char_loose_right: f32, + // Left edges of the same boxes. Right-to-left runs (Hebrew, Arabic) advance + // leftward, so their trailing edge is the LEFT one; `gap_to` picks the pair + // that matches the segment's writing direction. + last_char_left: f32, + last_char_loose_left: f32, + // Writing direction of the segment, latched from its first strong-direction + // character. Stays `None` through leading neutrals (digits, punctuation) so + // a segment opening with "(" still picks up RTL from the Hebrew/Arabic + // letter that follows. + dir_rtl: Option, // Bottom of last char strict bounds (for line-change detection) last_char_bottom: f32, // Count of non-space characters (for avg width calculation) @@ -2156,6 +2171,9 @@ impl SegmentBuilder { vp_bottom: f32::MIN, last_char_right: f32::MIN, last_char_loose_right: f32::MIN, + last_char_left: f32::MAX, + last_char_loose_left: f32::MAX, + dir_rtl: None, last_char_bottom: f32::MIN, char_count: 0, unmapped_char_count: 0, @@ -2233,6 +2251,48 @@ impl SegmentBuilder { self.word_has = false; } + /// Gap from the segment's last character to the incoming one, measured + /// along the writing direction against the previous char's *trailing* edge + /// (its right edge for left-to-right text, its left edge for right-to-left). + /// Positive means the incoming char sits further along the line, negative + /// means it doubled back — the same sign convention in both directions, so + /// every threshold downstream stays direction-agnostic. + fn gap_to(&self, vp_strict: &RectF, c: char) -> f32 { + if self.dir_is_rtl(c) { + self.last_char_left - vp_strict.right + } else { + vp_strict.left - self.last_char_right + } + } + + /// As [`gap_to`](Self::gap_to), but against the previous char's LOOSE box, + /// which subtracts out intra-word kerning/overhang. This is the + /// advance-relative gap the missing-space recovery compares to the font's + /// space width. + fn loose_gap_to(&self, vp_strict: &RectF, c: char) -> f32 { + if self.dir_is_rtl(c) { + self.last_char_loose_left - vp_strict.right + } else { + vp_strict.left - self.last_char_loose_right + } + } + + /// Writing direction to use for the pair (segment tail, `c`). Falls back to + /// the incoming character while the segment has only seen neutrals. + fn dir_is_rtl(&self, c: char) -> bool { + self.dir_rtl.unwrap_or_else(|| is_rtl_char(c)) + } + + /// Record `c`'s contribution to the segment's writing direction. Neutral + /// characters (digits, punctuation, spaces) leave it untouched. + fn note_direction(&mut self, c: char) { + if is_rtl_char(c) { + self.dir_rtl = Some(true); + } else if c.is_alphabetic() { + self.dir_rtl = Some(false); + } + } + /// Average width of non-space characters in the current segment. /// Prefers actual glyph widths (text_width) over bbox width, since bbox /// includes inter-character gaps that inflate the average and cause @@ -2266,7 +2326,11 @@ impl SegmentBuilder { self.vp_bottom = vp_loose.bottom; self.last_char_right = vp_strict.right; self.last_char_loose_right = vp_loose.right; + self.last_char_left = vp_strict.left; + self.last_char_loose_left = vp_loose.left; self.last_char_bottom = vp_strict.bottom; + self.dir_rtl = None; + self.note_direction(c); self.char_count = 1; self.unmapped_char_count = if counts_as_unmapped(recovered, ch.has_unicode_map_error()) { 1 @@ -2384,7 +2448,10 @@ impl SegmentBuilder { self.vp_bottom = self.vp_bottom.max(vp_loose.bottom); self.last_char_right = vp_strict.right; self.last_char_loose_right = vp_loose.right; + self.last_char_left = vp_strict.left; + self.last_char_loose_left = vp_loose.left; self.last_char_bottom = vp_strict.bottom; + self.note_direction(c); self.char_count += 1; if self.extract_text_metadata { self.char_codes.push(ch.char_code()); diff --git a/crates/liteparse/src/lib.rs b/crates/liteparse/src/lib.rs index 56dd89b9..56416da6 100644 --- a/crates/liteparse/src/lib.rs +++ b/crates/liteparse/src/lib.rs @@ -25,6 +25,7 @@ pub mod types; // ── Internal modules (available for binding crates, hidden from docs) ── #[cfg(not(target_arch = "wasm32"))] mod acroform_repair; +mod bidi; #[cfg(not(target_arch = "wasm32"))] #[doc(hidden)] pub mod conversion; diff --git a/crates/liteparse/src/markdown_layout/cross_region.rs b/crates/liteparse/src/markdown_layout/cross_region.rs index a0472ce2..77246c66 100644 --- a/crates/liteparse/src/markdown_layout/cross_region.rs +++ b/crates/liteparse/src/markdown_layout/cross_region.rs @@ -228,14 +228,28 @@ fn cluster_rows<'a>(set: &[(&'a ProjectedLine, bool)], tol: f32) -> Vec ProjectedLine { let mut members: Vec<&ProjectedLine> = cluster.members().copied().collect(); - members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x)); + // Fuse in reading order: a right-to-left row starts at its highest x, so + // walking left-to-right would emit its cells backwards. `fused.spans` is + // re-sorted x-ascending at the end either way, keeping cell geometry intact. + let rtl = crate::bidi::is_rtl_pieces(members.iter().map(|m| m.text.as_str())); + if rtl { + members.sort_by(|a, b| b.bbox.x.total_cmp(&a.bbox.x)); + } else { + members.sort_by(|a, b| a.bbox.x.total_cmp(&b.bbox.x)); + } let first = members[0]; let mut fused = first.clone(); + fused.rtl = rtl; fused.region_path = path.to_vec(); for m in &members[1..] { - let prev_right = fused.bbox.x + fused.bbox.width; - let gap = (m.bbox.x - prev_right).max(0.0); + // Gap to the previously-fused run, measured along the reading + // direction so the space count stays proportional in both. + let gap = if rtl { + (fused.bbox.x - (m.bbox.x + m.bbox.width)).max(0.0) + } else { + (m.bbox.x - (fused.bbox.x + fused.bbox.width)).max(0.0) + }; let approx_char_w = (fused.dominant_font_size * 0.5).max(2.0); let n_spaces = ((gap / approx_char_w) as usize).clamp(2, 24); fused.text.push_str(&" ".repeat(n_spaces)); diff --git a/crates/liteparse/src/markdown_layout/inline.rs b/crates/liteparse/src/markdown_layout/inline.rs index e0c1e0d7..e3751a5e 100644 --- a/crates/liteparse/src/markdown_layout/inline.rs +++ b/crates/liteparse/src/markdown_layout/inline.rs @@ -105,10 +105,17 @@ pub(super) fn render_line_inline(line: &ProjectedLine) -> String { return collapse_whitespace(&line.text); } - // Sort spans by x so we render in visual reading order regardless of - // extraction order. Stable so equal-x spans keep their original sequence. + // Sort spans into reading order regardless of extraction order. Stable so + // equal-x spans keep their original sequence. A right-to-left line reads + // from the highest x down, matching the join `build_one_line` used for + // `line.text` — the uniform-style shortcut below falls back on that string, + // so the two orderings have to agree. let mut spans = spans; - spans.sort_by(|a, b| a.x.total_cmp(&b.x)); + if line.rtl { + spans.sort_by(|a, b| b.x.total_cmp(&a.x)); + } else { + spans.sort_by(|a, b| a.x.total_cmp(&b.x)); + } let styles: Vec = spans.iter().map(|s| SpanStyle::from_item(s)).collect(); let links: Vec> = spans.iter().map(|s| s.link.as_deref()).collect(); diff --git a/crates/liteparse/src/markdown_layout/test_helpers.rs b/crates/liteparse/src/markdown_layout/test_helpers.rs index e350f25d..9f6874ba 100644 --- a/crates/liteparse/src/markdown_layout/test_helpers.rs +++ b/crates/liteparse/src/markdown_layout/test_helpers.rs @@ -5,6 +5,7 @@ use crate::types::{Anchor, GraphicPrimitive, ParsedPage, ProjectedLine, Rect, Te pub(crate) fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine { ProjectedLine { text: text.into(), + rtl: crate::bidi::is_rtl_text(text), bbox: Rect { x, y, @@ -84,6 +85,7 @@ pub(crate) fn line_with_spans(cells: &[(&str, f32)], y: f32, size: f32) -> Proje .map(|(t, _)| *t) .collect::>() .join(" "), + rtl: false, bbox: Rect { x: min_x, y, @@ -134,6 +136,7 @@ pub(crate) fn styled_line(spans: &[(&str, f32, Option<&str>)], y: f32, size: f32 .map(|s| s.x + s.width) .fold(f32::NEG_INFINITY, f32::max); ProjectedLine { + rtl: crate::bidi::is_rtl_text(&joined), text: joined, bbox: Rect { x: min_x, diff --git a/crates/liteparse/src/output/markdown.rs b/crates/liteparse/src/output/markdown.rs index 8d698050..79899d00 100644 --- a/crates/liteparse/src/output/markdown.rs +++ b/crates/liteparse/src/output/markdown.rs @@ -134,6 +134,7 @@ mod tests { fn line(text: &str, x: f32, y: f32, h: f32, size: f32) -> ProjectedLine { ProjectedLine { text: text.into(), + rtl: crate::bidi::is_rtl_text(text), bbox: Rect { x, y, diff --git a/crates/liteparse/src/projection.rs b/crates/liteparse/src/projection.rs index 6c22e020..2690dcae 100644 --- a/crates/liteparse/src/projection.rs +++ b/crates/liteparse/src/projection.rs @@ -4737,11 +4737,15 @@ fn build_one_line( figures: &[Rect], ) -> ProjectedLine { // Sort by x so concatenation reads left→right even if reading order had - // rotated insertions. + // rotated insertions. `spans` stays in this x-ascending order — the table + // cell splitter and the inline emphasis renderer both read geometry off it + // — while `text` below is joined in *reading* order, which for a + // right-to-left line runs the other way across the page. let mut sorted: Vec = idxs.to_vec(); sorted.sort_by(|a, b| items[*a].item.x.total_cmp(&items[*b].item.x)); - let mut text = String::new(); + // Item texts in x order; joined once the line's base direction is known. + let mut piece_texts: Vec<&str> = Vec::with_capacity(sorted.len()); let mut min_x = f32::INFINITY; let mut min_y = f32::INFINITY; let mut max_x = f32::NEG_INFINITY; @@ -4770,7 +4774,7 @@ fn build_one_line( let mut mcid: Option = None; let mut spans: Vec = Vec::with_capacity(sorted.len()); - for (pos, &i) in sorted.iter().enumerate() { + for &i in sorted.iter() { let proj = &items[i]; let it = &proj.item; // `handle_rotation_reading_order` zeroes `item.rotation` after it @@ -4783,13 +4787,10 @@ fn build_one_line( span.rotation = proj.orig_rotation; spans.push(span); - // Concatenate item text. Use existing num_spaces from projection only as + // Collect item text. Use existing num_spaces from projection only as // a hint — the markdown emitter re-collapses whitespace, so we just // ensure there's *some* separation between adjacent items. - if pos > 0 && !text.ends_with(' ') { - text.push(' '); - } - text.push_str(&it.text); + piece_texts.push(it.text.as_str()); min_x = min_x.min(it.x); min_y = min_y.min(it.y); @@ -4849,6 +4850,25 @@ fn build_one_line( } } + // Join the line in reading order. PDFium already hands back each item's + // characters in logical order, so a right-to-left line only needs its + // *items* walked right-to-left across the page — otherwise an invoice line + // like "المجموع الفرعي: 75.00 SAR" comes out as "SAR 75.00 المجموع الفرعي:", + // detaching every label from its value. Direction is decided from the + // assembled line, so a neutral-only line (pure digits) stays LTR and every + // left-to-right document takes exactly the path it took before. + let rtl = crate::bidi::is_rtl_pieces(piece_texts.iter().copied()); + if rtl { + piece_texts.reverse(); + } + let mut text = String::new(); + for piece in &piece_texts { + if !text.is_empty() && !text.ends_with(' ') { + text.push(' '); + } + text.push_str(piece); + } + // NOTE on tie-breaks: `max_by_key` over a HashMap returns the *last* max it // iterates, and HashMap iteration order is randomized per process. Ties on // char-weight would therefore pick a different winner every run, making the @@ -4947,6 +4967,7 @@ fn build_one_line( ProjectedLine { text, + rtl, bbox: bbox.clone(), anchor, // Real column detection is deferred (carry-forward in MARKDOWN_PROGRESS). diff --git a/crates/liteparse/src/types.rs b/crates/liteparse/src/types.rs index aacaee4f..df40150b 100644 --- a/crates/liteparse/src/types.rs +++ b/crates/liteparse/src/types.rs @@ -584,6 +584,11 @@ pub struct ProjectedLine { pub all_mono: bool, pub all_strike: bool, pub spans: Vec, + /// True when the line's base direction is right-to-left (strong RTL + /// characters outnumber strong LTR ones). `spans` is always in x-ascending + /// order; consumers that rebuild the line's *text* must walk it in reverse + /// when this is set, or labels detach from their values. + pub rtl: bool, /// Path from the page's region-tree root to the leaf containing this line. /// Equality means "same leaf"; prefix relationship means "one contains the /// other". Replaces the prior flat `column_id` scheme so nested layouts From 05f389b3737c0d75a7225ad2b5ab2333320dd483 Mon Sep 17 00:00:00 2001 From: Logan Markewich Date: Fri, 31 Jul 2026 20:19:57 -0600 Subject: [PATCH 05/50] add missing file --- crates/liteparse/src/bidi.rs | 135 +++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 crates/liteparse/src/bidi.rs diff --git a/crates/liteparse/src/bidi.rs b/crates/liteparse/src/bidi.rs new file mode 100644 index 00000000..a509a99f --- /dev/null +++ b/crates/liteparse/src/bidi.rs @@ -0,0 +1,135 @@ +//! Minimal bidirectional-text helpers. +//! +//! PDF content streams store right-to-left text in *logical* order already — +//! a correctly generated Hebrew/Arabic PDF places the first logical character +//! at the highest x and advances leftward, and PDFium hands the characters back +//! in that same stream order. So liteparse never needs to reverse characters; +//! what it does need is to stop assuming that "left to right on the page" means +//! "first to last in reading order". +//! +//! Two places care: +//! * `extract` measures inter-character gaps along the writing direction +//! (see `SegmentBuilder::gap_to`), so RTL runs are not sheared into +//! fragments by the line-change heuristics. +//! * line assembly concatenates a line's items in reading order, which for an +//! RTL line runs right-to-left across the page. + +/// Strong right-to-left character (Unicode bidi class R or AL). Covers Hebrew, +/// Arabic, Syriac, Thaana, N'Ko, Samaritan, Mandaic, Adlam and the Arabic +/// presentation-form blocks that subset fonts commonly map to. +pub(crate) fn is_rtl_char(c: char) -> bool { + matches!(c as u32, + 0x0590..=0x05FF // Hebrew + | 0x0600..=0x07BF // Arabic, Syriac, Thaana, N'Ko + | 0x0800..=0x085F // Samaritan, Mandaic + | 0x08A0..=0x08FF // Arabic Extended-A + | 0xFB1D..=0xFDFF // Hebrew/Arabic presentation forms A + | 0xFE70..=0xFEFF // Arabic presentation forms B + | 0x10800..=0x10FFF // Cypriot..Old Hungarian etc. + | 0x1E800..=0x1EFFF // Mende Kikakui, Adlam, Arabic Math + ) +} + +/// Strong left-to-right character. Deliberately *not* the complement of +/// [`is_rtl_char`]: digits, punctuation and whitespace are bidi-neutral and +/// must not vote on a line's base direction, or every "12.34 SAR" amount on an +/// Arabic invoice would drag its line back to LTR. +fn is_strong_ltr_char(c: char) -> bool { + c.is_alphabetic() && !is_rtl_char(c) +} + +/// Base direction of a run of characters, decided by majority of +/// strong-direction characters. Neutral-only text (pure numbers, punctuation) +/// is LTR, which keeps every left-to-right document on exactly the path it took +/// before. +fn is_rtl_chars(chars: impl Iterator) -> bool { + let mut rtl = 0usize; + let mut ltr = 0usize; + for c in chars { + if is_rtl_char(c) { + rtl += 1; + } else if is_strong_ltr_char(c) { + ltr += 1; + } + } + rtl > ltr +} + +/// Base direction of a line of text. See [`is_rtl_chars`]. +pub(crate) fn is_rtl_text(s: &str) -> bool { + !s.is_ascii() && is_rtl_chars(s.chars()) +} + +/// Base direction of a line still held as separate pieces, without joining +/// them first — line assembly needs the direction *before* it picks a join +/// order, and the pieces can be numerous. +/// +/// Every RTL character is non-ASCII, so a line whose pieces are all ASCII has +/// an RTL count of zero and cannot be RTL whatever its LTR count. `is_ascii` is +/// a byte-wise scan rather than per-`char` classification, which keeps this off +/// the hot path for left-to-right documents — the overwhelmingly common case. +pub(crate) fn is_rtl_pieces<'a>(pieces: impl IntoIterator + Clone) -> bool { + if pieces.clone().into_iter().all(|p| p.is_ascii()) { + return false; + } + is_rtl_chars(pieces.into_iter().flat_map(str::chars)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strong_direction_classification() { + assert!(is_rtl_char('\u{05DE}')); // Hebrew mem + assert!(is_rtl_char('\u{0642}')); // Arabic qaf + assert!(!is_rtl_char('A')); + assert!(!is_rtl_char('7')); + // Digits and punctuation are neutral, not strong LTR. + assert!(!is_strong_ltr_char('7')); + assert!(!is_strong_ltr_char('.')); + assert!(is_strong_ltr_char('A')); + } + + #[test] + fn line_direction_ignores_neutrals() { + // An Arabic label with a Latin currency code and an amount stays RTL: + // the digits must not outvote the letters. + assert!(is_rtl_text("المجموع الفرعي: 75.00")); + assert!(is_rtl_text("סך הכל 12.34")); + // A Latin line with a stray Hebrew word stays LTR. + assert!(!is_rtl_text("Total due 87.75 ILS")); + // Neutral-only lines stay LTR so LTR documents are untouched. + assert!(!is_rtl_text("2026-07-27 10:30")); + assert!(!is_rtl_text("")); + } + + #[test] + fn ascii_fast_path_agrees_with_full_scan() { + // The `is_ascii` short-circuit must be an optimization only: for any + // input it has to return exactly what the per-char scan would. + for s in [ + "", + "Total due 87.75 ILS", + "2026-07-27", + "!@#$%^&*()", + "\u{05DE}\u{05E1}", + "mixed \u{0642} latin", + "caf\u{e9} na\u{ef}ve", + ] { + assert_eq!(is_rtl_text(s), is_rtl_chars(s.chars()), "mismatch on {s:?}"); + assert_eq!(is_rtl_pieces([s]), is_rtl_chars(s.chars())); + } + // Split across pieces, where no single piece is decisive. + assert_eq!( + is_rtl_pieces(["12.34 ", "\u{05DE}\u{05E1}\u{05E2}"]), + is_rtl_chars("12.34 \u{05DE}\u{05E1}\u{05E2}".chars()) + ); + } + + #[test] + fn currency_code_does_not_flip_short_rtl_line() { + // "USD" is 3 strong LTR chars; the Hebrew must still win. + assert!(is_rtl_text("סך הכל לתשלום 87.75 ILS")); + } +} From 093ddba792cbbf3b04cf09fafdebfb7ee5e39314 Mon Sep 17 00:00:00 2001 From: Logan Markewich Date: Sun, 2 Aug 2026 08:37:00 -0600 Subject: [PATCH 06/50] table improvements --- .../liteparse/src/markdown_layout/tables.rs | 159 +++++++++++++++++- 1 file changed, 158 insertions(+), 1 deletion(-) diff --git a/crates/liteparse/src/markdown_layout/tables.rs b/crates/liteparse/src/markdown_layout/tables.rs index b0ca3b27..bad07932 100644 --- a/crates/liteparse/src/markdown_layout/tables.rs +++ b/crates/liteparse/src/markdown_layout/tables.rs @@ -1076,7 +1076,35 @@ fn absorb_header_lines( let mut j = start_idx; while j > floor { let cand = j - 1; - let cells = split_cells(&lines[cand]); + let mut cells = split_cells(&lines[cand]); + // PDFium routinely emits a header row's words as one merged text run, + // so an 8-column table arrives with a 2-cell header whose second cell + // spans seven tracks. `match_track_idx` below can only bind that blob + // to a single column, collapsing the whole header band into one cell. + // Body rows already get merged-run recovery; headers need it too. + // + // Prose carries whitespace everywhere, so recovery "succeeds" on a + // paragraph sitting above the table just as readily as on a real + // header — manufacturing a header out of prose and dragging the + // paragraph into the table. Cell count can't separate them (a projected + // prose line splits on its own internal gaps), but length can: header + // labels are terse, shredded prose is not. + // + // Restricted to the first candidate (the line directly above the body): + // a merged-run header is a single line, whereas letting recovery + // track-align line after line lets absorption climb an entire + // paragraph, one plausible-looking row at a time. + if absorbed.is_empty() && cells.len() < column_count { + const HEADER_MAX_CELL_CHARS: usize = 30; + let tracks: Vec = track_ranges.iter().map(|r| r.0).collect(); + if let Some(recovered) = recover_merged_cell(cells.clone(), &tracks) + && recovered + .iter() + .all(|c| c.text.chars().count() <= HEADER_MAX_CELL_CHARS) + { + cells = recovered; + } + } if dbgt { let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect(); eprintln!( @@ -3246,6 +3274,80 @@ fn passes_density_gate( /// Build a `TableRun` for one ruled-grid component. Returns `None` if the /// resulting grid is too small (< 2 cols or < 2 rows), covers nearly the /// whole page (likely the page border), or is mostly empty cells. +/// Fuse "gutter" column boundaries left behind by paired cell-border edges. +/// +/// A bordered cell contributes two vertical edges — its own right edge and the +/// next cell's left edge — separated by the table's inter-cell padding. +/// `TABLE_COL_BOUNDARY_CLUSTER_PT` fuses the tight pairs (4-6pt), but a +/// generously padded table puts 15-25pt between them, leaving a sliver column +/// between every real column. Those slivers double the column count, and +/// because `split_span_at_anchors` shreds text into them they are *not* empty, +/// so `collapse_phantom_cols` can't remove them — the grid then fails the +/// empty-cell gate and a perfectly good table is thrown away. +/// +/// Width alone can't identify them — plenty of real tables carry a genuinely +/// narrow column (a `#` or `No.` column beside a wide description). What marks +/// a sliver is that **no text center ever lands inside it**: text steps over a +/// gutter, but a narrow real column still holds its own values. So fuse a +/// column only when it holds no span center *and* is far narrower than the +/// columns that do hold text. Span centers are read straight off the raw spans, +/// before `split_span_at_anchors` runs, so shredded fragments can't disguise a +/// sliver as occupied. +/// +/// The width guard matters independently: a worksheet's blank answer column is +/// also centerless, but it is as wide as its neighbours and must survive. +fn collapse_gutter_columns(xs: &mut Vec, lines: &[ProjectedLine], dbg: bool) { + /// A sliver must be at most this fraction of the median *content-bearing* + /// column width. Blank-but-full-width cells (worksheets, forms) sit well + /// above it and are preserved. + const GUTTER_MAX_WIDTH_FRAC: f32 = 0.5; + + if xs.len() < 4 { + return; // fewer than 3 columns: nothing to fuse without destroying the table + } + let centers: Vec = lines + .iter() + .flat_map(|l| l.spans.iter()) + .filter(|s| !s.text.trim().is_empty()) + .map(|s| s.x + s.width * 0.5) + .collect(); + let has_center = |lo: f32, hi: f32| centers.iter().any(|&c| c >= lo && c < hi); + + let before = xs.len(); + while xs.len() > 3 { + // Median width of the columns that actually hold text. + let mut occupied: Vec = xs + .windows(2) + .filter(|w| has_center(w[0], w[1])) + .map(|w| w[1] - w[0]) + .collect(); + if occupied.is_empty() { + break; + } + occupied.sort_by(|a, b| a.total_cmp(b)); + let median = occupied[occupied.len() / 2]; + let max_sliver = median * GUTTER_MAX_WIDTH_FRAC; + + // Narrowest centerless column, if any is slim enough to be a gutter. + let victim = xs + .windows(2) + .enumerate() + .filter(|(_, w)| !has_center(w[0], w[1])) + .map(|(i, w)| (i, w[1] - w[0])) + .filter(|&(_, width)| width < max_sliver) + .min_by(|a, b| a.1.total_cmp(&b.1)); + let Some((i, _)) = victim else { break }; + xs[i] = (xs[i] + xs[i + 1]) * 0.5; + xs.remove(i + 1); + } + if dbg && xs.len() != before { + eprintln!( + "[ruled] gutter-collapse {before} -> {} boundaries", + xs.len() + ); + } +} + fn build_ruled_table( hs: &[HSeg], vs: &[VSeg], @@ -3263,6 +3365,7 @@ fn build_ruled_table( // rects contribute paired edges 4-6pt apart that would otherwise become // phantom 5pt "columns" the span splitter then shreds text into. cluster_boundaries(&mut xs, TABLE_COL_BOUNDARY_CLUSTER_PT); + collapse_gutter_columns(&mut xs, lines, dbg); // Distinct row y-coords (cluster again — multiple H lines may share a y). // In the global pass, first drop horizontal rules that span only a small @@ -3721,6 +3824,60 @@ mod tests { use super::super::test_helpers::{line, line_with_spans, rect_borders, stroke}; use super::*; + #[test] + fn gutter_columns_fused_but_narrow_real_column_kept() { + // Padded cell borders: real columns 40..140, 160..260, 280..380 with + // 20pt gutters between them. Text sits inside the real columns only. + let rows = [ + line_with_spans( + &[("alpha", 45.0), ("beta", 165.0), ("gamma", 285.0)], + 100.0, + 10.0, + ), + line_with_spans( + &[("delta", 45.0), ("eps", 165.0), ("zeta", 285.0)], + 120.0, + 10.0, + ), + ]; + let mut xs = vec![40.0, 140.0, 160.0, 260.0, 280.0, 380.0]; + collapse_gutter_columns(&mut xs, &rows, false); + assert_eq!(xs.len(), 4, "two 20pt gutters should fuse: {xs:?}"); + + // A genuinely narrow but *occupied* column must survive, even though it + // is the same width as the gutters above. + let rows = [ + line_with_spans( + &[("1", 45.0), ("desc", 65.0), ("more text", 205.0)], + 100.0, + 10.0, + ), + line_with_spans( + &[("2", 45.0), ("desc two", 65.0), ("text here", 205.0)], + 120.0, + 10.0, + ), + ]; + let mut xs = vec![40.0, 60.0, 200.0, 380.0]; + let before = xs.clone(); + collapse_gutter_columns(&mut xs, &rows, false); + assert_eq!(xs, before, "occupied narrow column must not fuse"); + } + + #[test] + fn blank_worksheet_column_survives_gutter_collapse() { + // A worksheet's empty answer column is centerless like a gutter but is + // full width, so the width guard must keep it. + let rows = [ + line_with_spans(&[("K+", 45.0)], 100.0, 10.0), + line_with_spans(&[("Na+", 45.0)], 120.0, 10.0), + ]; + let mut xs = vec![40.0, 240.0, 440.0, 640.0]; + let before = xs.clone(); + collapse_gutter_columns(&mut xs, &rows, false); + assert_eq!(xs, before, "wide blank columns must not fuse: {xs:?}"); + } + #[test] fn split_cells_splits_on_wide_gaps() { let l = line_with_spans(&[("A", 50.0), ("B", 150.0), ("C", 250.0)], 100.0, 10.0); From 520413d04c4136d542549648e48b9b355410b393 Mon Sep 17 00:00:00 2001 From: Logan Markewich Date: Sun, 2 Aug 2026 12:22:40 -0600 Subject: [PATCH 07/50] table improvements: two-row tables + soft-wrap row grouping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes followups 1 and 2 in TABLE_TEDS_PROGRESS.md. On opendataloader-bench TEDS goes 0.7128 -> 0.7441, cutting the gap to pdf-inspector from 0.101 to 0.070; overall 0.8757 -> 0.8783. Neutral-to-positive on the ParseBench table dimension (0.4032 -> 0.4034). Two-pass last-resort for header + single-data-row tables. try_detect_table_ inferred takes allow_two_row; two_row_second_pass retries only in the index gaps where the normal pass found no table, so it cannot steal a real table's header. The gap restriction alone is not sufficient — fully-justified prose infers clean tracks from its stretched inter-word spaces, so two_row_run_ plausible also requires isolation and header shape. Doc 197: 0.000 -> 0.789, one doc changed, no collateral. Soft-wrap row grouping (merge_continuation_rows), applied where ruled and borderless runs converge. A row folds into its predecessor when it has an empty first cell, its filled columns are a subset of the predecessor's, and it carries no value-like cell. Guarded by a first-column fill test: "empty first cell" only means "wrapped line" when the first column is a label column, and without it sparse-first-column tables (timetables, size charts) collapse to a single row. Doc 150: 0.446 -> 0.861. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013TKUnWQHGdXrcP9KNSsckd --- TABLE_TEDS_PROGRESS.md | 197 +++++++++++ .../liteparse/src/markdown_layout/tables.rs | 319 +++++++++++++++++- 2 files changed, 510 insertions(+), 6 deletions(-) create mode 100644 TABLE_TEDS_PROGRESS.md diff --git a/TABLE_TEDS_PROGRESS.md b/TABLE_TEDS_PROGRESS.md new file mode 100644 index 00000000..0e44f966 --- /dev/null +++ b/TABLE_TEDS_PROGRESS.md @@ -0,0 +1,197 @@ +# Table / TEDS gap vs pdf-inspector — progress & followups + +Goal: close the table-fidelity (TEDS) gap against **pdf-inspector** (firecrawl) on +`opendataloader-bench` (200 PDFs). Started 2026-08-01. + +## Standing + +| metric | baseline (2.10.1) | round 1 (`093ddba`) | round 2 | Δ total | pdf-inspector | +|---|---|---|---|---|---| +| **overall** | 0.8732 | 0.8757 | **0.8783** | +0.0051 | 0.8753 | +| TEDS | 0.6929 | 0.7128 | **0.7441** | **+0.0511** | 0.8141 | +| MHS | 0.8114 | 0.8175 | **0.8179** | +0.0064 | 0.7879 | +| NID | 0.9127 | 0.9129 | **0.9136** | +0.0009 | 0.9147 | + +We lead on overall + MHS, trail on TEDS by **0.070** (was 0.101) and NID by 0.001. +**TEDS is scored on only 42 of the 200 docs**, so one doc = 0.024 of the mean. + +Round 1 shipped as `093ddba`; round 2 adds ~190 lines, all still in +`crates/liteparse/src/markdown_layout/tables.rs`. 298 lib tests pass. + +## What shipped + +1. **`collapse_gutter_columns`** — called from `build_ruled_table` right after + `cluster_boundaries`. Padded cell borders contribute *paired* vertical edges + (15–25pt apart), leaving a sliver column between every real column — doc 200 had 13 + columns for a 4-column table. `split_span_at_anchors` shreds text into the slivers, so + `collapse_phantom_cols` can't see they're empty and the grid dies on the empty-cell + gate. Fuse a column only when **no raw span center lands inside it** AND it is + < 0.5× the median *content-bearing* column width. + *Both halves are load-bearing*: a width-only/bimodal variant broke docs 178+120 from + 1.000 → 0.65, and the width guard is what preserves blank worksheet columns. + +2. **Merged-run recovery for absorbed headers** — in `absorb_header_lines`. PDFium emits + a header row's words as one run, so an 8-col table got a 2-cell header whose blob + bound to a single column (doc 190). Reuses the existing `recover_merged_cell`. + Needs **three** gates or it shreds prose into fake headers: + `absorbed.is_empty()` (first candidate line only), every recovered piece ≤ 30 chars, + and `cells.len() < column_count`. Dropping any one of these regressed NID. + +## What shipped — round 2 (followups 1 + 2) + +3. **Two-pass last-resort for header + single-data-row tables** (+0.0188 TEDS, + +0.0002 NID; doc 197 alone 0.000 → 0.789, *one* doc changed, no collateral). + `try_detect_table_inferred` takes `allow_two_row`; `two_row_second_pass` retries + only in the index gaps where the normal pass found no table. + *The gap restriction alone is NOT enough* — that was the plan and it failed: + doc 147 still broke (−0.82) and 14 docs lost NID. The real counterfeit is + **fully-justified prose**, whose stretched inter-word spaces infer as clean + tracks, so any two consecutive lines shred into + `| when travelling | to | conflict | zones, | more |`. What actually works is + `two_row_run_plausible`: **isolation** (neighbours not table-adjacent — a prose + pair is mid-paragraph) plus **header shape** (cells start uppercase/digit, no + trailing comma). Gates apply only when the relaxation actually fired, so they + can never reject something the normal pass accepted. + +4. **Multi-line cell / row grouping** (+0.0125 TEDS, 3 docs up, none down; + doc 150 0.446 → 0.861, now *beating* pdf-inspector's 0.847). + `merge_continuation_rows`, applied at the end of `merge_table_runs` — the one + funnel both ruled and borderless runs pass through. Post-hoc and text-only, as + predicted; the geometry really is unusable. A row folds into its predecessor + when it has an empty first cell, its filled columns are a **subset** of the + predecessor's, and at least one extended cell reads as a soft wrap. + Two vetoes are load-bearing, and **each was found by a different benchmark**: + - **Any value-like cell → not a continuation** (odl-bench). Numbers don't + soft-wrap, and a `Total` row has a blank label column precisely because the + label isn't its own. Vetoing on `all` value-like instead of `any` cost docs + 45 (−0.13) and 47 (−0.09); `any` turned both positive. + - **First column must be filled in ≥ half the rows** (ParseBench only — + see the cross-benchmark note below). "Empty first cell" only means "wrapped + line" if the first column is a *label* column; plenty of tables just have a + sparse one. + +## Cross-benchmark check is mandatory for table work + +**opendataloader-bench alone is not enough.** Round 2 looked perfect on it — four +docs up, none down — while silently costing **−0.0048** on the ParseBench table +dimension. Attribution (one run per variant): + +| variant | odl TEDS | ParseBench table composite | +|---|---|---| +| HEAD `093ddba` | 0.7128 | 0.4032 | +| + two-row pass | 0.7316 | 0.4032 (free) | +| + continuation merge | 0.7441 | **0.3984** ← regression | +| + first-column fill guard | 0.7441 | **0.4034** | + +The continuation merge owned the whole regression, concentrated in a few docs +(`1 timetable_page6` 0.859 → **0.006**, `sizingchart`, `myco hierarchical table +header`). All were tables with a legitimately sparse first column, where every +row reads as a continuation and the table collapses to one row. The fill guard +fixes them at zero cost to odl-bench. + +Note `is_value_like` does **not** match bare times (`8:24`) or bare integers +(`15`) — it needs `d.d` / `d,d` / `$` / `%`. Don't widen it to patch a table bug; +it gates several unrelated paths. Prefer a structural guard. + +Run: `cd ParseBench && uv run parse-bench run liteparse_markdown --group table` +(~12 min, 503 examples). Read `avg_grits_trm_composite` from +`output/liteparse_markdown/_evaluation_report.json` — this is the leaderboard's +"Tables" column (0.4032 = the 40.3 in `leaderboard.csv`). **The run overwrites +that report in place**, so copy it aside before running a variant. + +## Tooling + +- **Per-doc deltas**: `opendataloader-bench/perdoc.py [label-a] [metric]` + — prints every doc whose metric moved between two prediction labels, worst + first. Doc numbers match the ones used in this file. This is what caught that + the round-2 mean hid a −0.82 doc in the first two-row attempt. +- **A/B harness**: `opendataloader-bench/ab.sh