diff --git a/README.md b/README.md index 8be97580..f4ac665f 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,12 @@ wasm/ — Browser bindings (wasm-bindgen) This detects 300+ page PDFs in milliseconds. The result includes `pages_needing_ocr` — a list of specific page numbers that lack text, enabling per-page OCR routing instead of all-or-nothing. +Fast `detect_pdf*` and `classify_pdf*` calls inspect structural signals only; +they do not extract text or validate font/character encoding. Therefore, +`pages_needing_ocr = []` is not a text-quality verdict. Use +`ProcessMode::Analyze`, `process_pdf*`, or `extract_pages_markdown*` when OCR +routing must also catch broken or garbled text encodings. + ### Scan strategies | Strategy | Behavior | Best for | diff --git a/docs/python.md b/docs/python.md index 242c23f3..b436a9c2 100644 --- a/docs/python.md +++ b/docs/python.md @@ -115,6 +115,13 @@ headings = [ ] ``` +`detect_pdf*` and `classify_pdf*` are fast structural classifiers. They do not +extract text or validate font/character encoding, so +`pages_needing_ocr == []` is not a text-quality verdict. For OCR routing that +includes broken or garbled encodings, use `extract_pages_markdown*` or the full +`process_pdf*` APIs. Use `process_pdf_with_ocr*` when those routing decisions +should also run selective OCR. + ## API reference | Function | Description | @@ -123,10 +130,10 @@ headings = [ | `process_pdf_bytes(data, pages=None)` | Full processing from bytes | | `process_pdf_with_ocr(path, **options)` | Native extraction + selective OCR with provenance | | `process_pdf_with_ocr_bytes(data, **options)` | Native extraction + selective OCR from bytes | -| `detect_pdf(path)` | Fast detection only (returns PdfResult) | -| `detect_pdf_bytes(data)` | Fast detection from bytes | -| `classify_pdf(path)` | Lightweight classification (returns PdfClassification) | -| `classify_pdf_bytes(data)` | Lightweight classification from bytes | +| `detect_pdf(path)` | Fast structural detection only; no text-quality analysis (returns PdfResult) | +| `detect_pdf_bytes(data)` | Fast structural detection from bytes; no text-quality analysis | +| `classify_pdf(path)` | Lightweight structural classification; no text-quality analysis (returns PdfClassification) | +| `classify_pdf_bytes(data)` | Lightweight structural classification from bytes; no text-quality analysis | | `extract_text(path)` | Plain text extraction | | `extract_text_bytes(data)` | Plain text extraction from bytes | | `extract_text_with_positions(path, pages=None)` | Text with X/Y coords and font info | @@ -203,7 +210,7 @@ class OcrPdfResult: # process_pdf_with_ocr / bytes class PdfClassification: # classify_pdf pdf_type: str page_count: int - pages_needing_ocr: list[int] # 0-indexed + pages_needing_ocr: list[int] # 0-indexed structural signals only confidence: float class TextItem: # extract_text_with_positions diff --git a/docs/rust-api.md b/docs/rust-api.md index 13ed38ac..17930017 100644 --- a/docs/rust-api.md +++ b/docs/rust-api.md @@ -81,6 +81,12 @@ match info.pdf_type { } ``` +`detect_pdf*` and `classify_pdf_mem` only inspect structural signals. They do +not extract text or validate font/character encoding, so an empty +`pages_needing_ocr` is not a text-quality verdict. Use `ProcessMode::Analyze` +to run extraction and text-quality routing without generating Markdown, or use +`extract_pages_markdown*` when per-page Markdown is also needed. + Customize processing with `PdfOptions`: ```rust @@ -492,18 +498,19 @@ for item in extract_text_with_positions("tagged.pdf")? { | Mode | What it does | Returns | |---|---|---| | `ProcessMode::Full` (default) | Detect + extract + convert to Markdown | Everything populated | -| `ProcessMode::Analyze` | Detect + extract + layout analysis (no Markdown) | `markdown` is `None`, `layout` is populated | -| `ProcessMode::DetectOnly` | Classification only (fastest) | `markdown` is `None`, `layout` is default | +| `ProcessMode::Analyze` | Detect + extract + text-quality/layout analysis (no Markdown) | `markdown` is `None`, `layout` and encoding-quality OCR routing are populated | +| `ProcessMode::DetectOnly` | Structural classification only (fastest; no text-quality analysis) | `markdown` is `None`, `layout` is default | ## Functions | Function | Description | |---|---| | `process_pdf(path)` | Full processing with defaults | -| `detect_pdf(path)` | Fast metadata-only detection (no extraction) | +| `detect_pdf(path)` | Fast structural detection (no extraction or text-quality analysis) | | `process_pdf_with_options(path, options)` | Process with custom `PdfOptions` | | `process_pdf_mem(bytes)` | Full processing from a byte buffer | -| `detect_pdf_mem(bytes)` | Fast detection from a byte buffer | +| `detect_pdf_mem(bytes)` | Fast structural detection from a byte buffer | +| `classify_pdf_mem(bytes)` | Lightweight structural classification; 0-indexed OCR pages, no text-quality analysis | | `process_pdf_mem_with_options(bytes, options)` | Process from bytes with custom options | | `extract_text(path)` | Plain text extraction | | `extract_text_with_positions(path)` | Text with X/Y coordinates and font info | diff --git a/pdf_inspector.pyi b/pdf_inspector.pyi index c80a4a9c..87796384 100644 --- a/pdf_inspector.pyi +++ b/pdf_inspector.pyi @@ -75,12 +75,18 @@ class OcrPdfResult: ocr_time_ms: int class PdfClassification: - """Lightweight PDF classification result.""" + """Lightweight structural PDF classification result. + + This result does not include extracted-text encoding-quality analysis. + """ pdf_type: str """'text_based', 'scanned', 'image_based', or 'mixed'.""" page_count: int pages_needing_ocr: list[int] - """0-indexed page numbers that need OCR.""" + """0-indexed pages with structural OCR signals. + + An empty list does not assert that extracted text has a valid encoding. + """ confidence: float class TextItem: @@ -204,11 +210,11 @@ def detect_pdf_bytes(data: bytes) -> PdfResult: ... def classify_pdf(path: str) -> PdfClassification: - """Lightweight classification — type, page count, and OCR pages (0-indexed).""" + """Lightweight structural classification without text-quality analysis.""" ... def classify_pdf_bytes(data: bytes) -> PdfClassification: - """Lightweight classification from bytes.""" + """Lightweight structural classification without text-quality analysis.""" ... def extract_text(path: str) -> str: diff --git a/src/lib.rs b/src/lib.rs index f62f168b..a3fec7d3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -381,14 +381,21 @@ pub struct PdfClassification { pub pdf_type: PdfType, /// Total page count. pub page_count: u32, - /// 0-indexed page numbers that need OCR (scanned/image pages). + /// 0-indexed page numbers that need OCR based on structural signals such + /// as scanned/image pages. Lightweight classification does not extract + /// text, so an empty list is not a verdict on text encoding quality. pub pages_needing_ocr: Vec, /// Detection confidence score (0.0–1.0). pub confidence: f32, } /// Classify a PDF from a memory buffer without extracting text. -/// Returns the PDF type and which pages need OCR (~10-50ms). +/// Returns the PDF type and pages with structural OCR signals (~10-50ms). +/// +/// This fast path does not validate extracted-text encoding quality. Use +/// [`extract_pages_markdown_mem`] when encoding-quality OCR routing is needed, +/// or [`process_pdf_mem_with_options`] with [`ProcessMode::Analyze`] to run +/// text-quality analysis without generating Markdown. pub fn classify_pdf_mem(buffer: &[u8]) -> Result { validate_pdf_bytes(buffer)?; let (doc, page_count) = load_document_from_mem(buffer)?; @@ -6505,6 +6512,13 @@ mod tests { } } + fn test_text_item_with_font(page: u32, font: &str, text: &str) -> TextItem { + TextItem { + font: font.to_string(), + ..test_text_item_on_page(page, text) + } + } + fn test_image_item(width: f32, height: f32) -> TextItem { TextItem { item_type: ItemType::Image, @@ -6777,6 +6791,23 @@ mod tests { .collect() } + fn digit_heavy_caesar_garble() -> String { + let mut garbled = String::new(); + let mut letters = 0usize; + for ch in caesar_shift(CAESAR_PROSE, 8).chars() { + if ch.is_ascii_alphanumeric() || ch.is_whitespace() { + garbled.push(ch); + } + if ch.is_ascii_alphabetic() { + letters += 1; + if letters.is_multiple_of(17) { + garbled.push('7'); + } + } + } + garbled + } + #[test] fn test_mixed_case_caesar_shift_flagged() { assert!(detect_encoding_issues(&caesar_shift(CAESAR_PROSE, 3))); @@ -6857,6 +6888,246 @@ mod tests { assert_eq!(quality.pages_needing_ocr, vec![1]); } + #[test] + fn test_text_quality_flags_garbled_font_on_mixed_page() { + let healthy = CAESAR_PROSE.repeat(3); + let garbled = caesar_shift(CAESAR_PROSE, 8); + let items = vec![ + test_text_item_with_font(1, "Helvetica", &healthy), + test_text_item_with_font(1, "FixtureFont", &garbled), + ]; + + let quality = analyze_text_quality(&items); + + assert_eq!(quality.pages_needing_ocr, vec![1]); + assert_eq!( + quality.reasons_by_page.get(&1).cloned(), + Some(vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()]) + ); + } + + #[test] + fn test_text_quality_allows_clean_multi_font_page() { + let items = vec![ + test_text_item_with_font(1, "Helvetica", CAESAR_PROSE), + test_text_item_with_font(1, "TimesRoman", &CAESAR_PROSE.repeat(2)), + ]; + + let quality = analyze_text_quality(&items); + + assert!(!quality.has_encoding_issues); + assert!(quality.pages_needing_ocr.is_empty()); + } + + #[test] + fn test_text_quality_allows_structured_ascii_in_dedicated_fonts() { + let healthy = CAESAR_PROSE.repeat(3); + let mixed_case_code = "myXMLParser ".repeat(20); + let uppercase_code = "MYXMLPARSER ".repeat(20); + let items = vec![ + test_text_item_with_font(1, "Helvetica", &healthy), + test_text_item_with_font(1, "CodeFont", &mixed_case_code), + test_text_item_with_font(2, "Helvetica", &healthy), + test_text_item_with_font(2, "AcronymFont", &uppercase_code), + ]; + + let quality = analyze_text_quality(&items); + + assert!(!quality.has_encoding_issues); + assert!(quality.pages_needing_ocr.is_empty()); + } + + const BASE64_BINARY_SAMPLE: &str = + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w=="; + const BASE64_PROSE_SAMPLE: &str = + "VGhlIHJlZ2lzdHJhbnQgaGVyZWJ5IGFncmVlcyB0byBmdXJuaXNoIGEgY29weSBvZiBhbnkgc3VjaCBpbnN0cnVtZW50IHRvIHRoZSBDb21taXNzaW9uIHVwb24gcmVxdWVzdC4gVGhpcyBjZXJ0aWZpY2F0ZSBvZiBkZXNpZ25hdGlvbnMgd2FzIGZpbGVkIEZlYnJ1YXJ5IHdpdGggcmVzcGVjdCB0byBTZXJpZXMgUHJlZmVycmVkIFN0b2NrIGFuZCBpbmNvcnBvcmF0ZWQgaGVyZWluIGJ5IHJlZmVyZW5jZSB0byB0aGUgYW5udWFsIHJlcG9ydCBvbiBmb3JtIGZvciB0aGUgcGVyaW9kIGVuZGVkIERlY2VtYmVyIGFzIGFtZW5kZWQgYW5kIHJlc3RhdGVkIHRoZXJlYWZ0ZXIu"; + + #[test] + fn test_text_quality_allows_base64_in_dedicated_font() { + let healthy = CAESAR_PROSE.repeat(3); + let mut items = Vec::new(); + for (index, base64) in [BASE64_BINARY_SAMPLE, BASE64_PROSE_SAMPLE] + .into_iter() + .enumerate() + { + let page = index as u32 + 1; + items.push(test_text_item_with_font(page, "Helvetica", &healthy)); + items.push(test_text_item_with_font(page, "DataFont", base64)); + } + + let quality = analyze_text_quality(&items); + + assert!(!quality.has_encoding_issues); + assert!(quality.pages_needing_ocr.is_empty()); + } + + #[test] + fn test_text_quality_allows_chunked_base64_in_dedicated_font() { + // PDF content streams may split one Base64 token into many Tj/TJ + // fragments. Classification must not depend on those item boundaries. + let healthy = CAESAR_PROSE.repeat(3); + let mut items = Vec::new(); + for (index, base64) in [BASE64_BINARY_SAMPLE, BASE64_PROSE_SAMPLE] + .into_iter() + .enumerate() + { + let page = index as u32 + 1; + items.push(test_text_item_with_font(page, "Helvetica", &healthy)); + items.extend(base64.as_bytes().chunks(16).map(|chunk| { + test_text_item_with_font( + page, + "DataFont", + std::str::from_utf8(chunk).expect("Base64 is ASCII"), + ) + })); + } + + let quality = analyze_text_quality(&items); + + assert!(!quality.has_encoding_issues); + assert!(quality.pages_needing_ocr.is_empty()); + } + + #[test] + fn test_text_quality_allows_line_wrapped_base64_in_dedicated_font() { + // MIME-style Base64 wraps at 76 characters. Those long encoded lines + // are not the explicitly delimited short words required for a + // digit/symbol-heavy font sample to be classified as prose. + let wrapped = BASE64_PROSE_SAMPLE + .as_bytes() + .chunks(76) + .map(|chunk| std::str::from_utf8(chunk).expect("Base64 is ASCII")) + .collect::>() + .join("\n"); + let healthy = CAESAR_PROSE.repeat(3); + let items = vec![ + test_text_item_with_font(1, "Helvetica", &healthy), + test_text_item_with_font(1, "DataFont", &wrapped), + ]; + + let quality = analyze_text_quality(&items); + + assert!(!quality.has_encoding_issues); + assert!(quality.pages_needing_ocr.is_empty()); + } + + #[test] + fn test_text_quality_allows_short_wrapped_base64_in_dedicated_font() { + let healthy = CAESAR_PROSE.repeat(3); + let mut items = Vec::new(); + for (index, base64) in [BASE64_BINARY_SAMPLE, BASE64_PROSE_SAMPLE] + .into_iter() + .enumerate() + { + let page = index as u32 + 1; + let wrapped = base64 + .as_bytes() + .chunks(16) + .map(|chunk| std::str::from_utf8(chunk).expect("Base64 is ASCII")) + .collect::>() + .join(" "); + items.push(test_text_item_with_font(page, "Helvetica", &healthy)); + items.push(test_text_item_with_font(page, "DataFont", &wrapped)); + } + + let quality = analyze_text_quality(&items); + + assert!(!quality.has_encoding_issues); + assert!(quality.pages_needing_ocr.is_empty()); + } + + #[test] + fn test_text_quality_flags_spaceless_garbled_font_on_mixed_page() { + // Removing separators must not make genuine shifted prose look like a + // structured token and bypass the per-font detector. + let healthy = CAESAR_PROSE.repeat(3); + let spaceless_garble: String = SHIFTED_CIPHER_TEXT + .chars() + .filter(char::is_ascii_alphabetic) + .collect(); + let items = vec![ + test_text_item_with_font(1, "Helvetica", &healthy), + test_text_item_with_font(1, "BrokenFont", &spaceless_garble), + ]; + + let quality = analyze_text_quality(&items); + + assert_eq!(quality.pages_needing_ocr, vec![1]); + } + + #[test] + fn test_text_quality_flags_digit_heavy_garbled_font_on_mixed_page() { + // Broken maps can also shift identifiers and quantities. Keep the + // real word boundaries but remove punctuation and inject enough + // digits to satisfy the former broad Base64 exemption. + let digit_heavy_garble = digit_heavy_caesar_garble(); + let healthy = CAESAR_PROSE.repeat(3); + let items = vec![ + test_text_item_with_font(1, "Helvetica", &healthy), + test_text_item_with_font(1, "BrokenIdFont", &digit_heavy_garble), + ]; + + let quality = analyze_text_quality(&items); + + assert_eq!(quality.pages_needing_ocr, vec![1]); + } + + #[test] + fn test_text_quality_flags_word_item_garbled_font_on_mixed_page() { + // Some PDFs emit each visible word as its own text-showing item + // without carrying a trailing space in the extracted string. + let healthy = CAESAR_PROSE.repeat(3); + let digit_heavy_garble = digit_heavy_caesar_garble(); + let mut items = vec![test_text_item_with_font(1, "Helvetica", &healthy)]; + items.extend( + digit_heavy_garble + .split_whitespace() + .map(|word| test_text_item_with_font(1, "BrokenIdFont", word)), + ); + + let quality = analyze_text_quality(&items); + + assert_eq!(quality.pages_needing_ocr, vec![1]); + } + + #[test] + fn test_text_quality_flags_uniform_case_garbled_fonts_on_mixed_pages() { + let healthy = CAESAR_PROSE.repeat(3); + let lowercase_garble = caesar_shift(&CAESAR_PROSE.to_lowercase(), 5); + let uppercase_garble = caesar_shift(&CAESAR_PROSE.to_uppercase(), 7); + let items = vec![ + test_text_item_with_font(1, "Helvetica", &healthy), + test_text_item_with_font(1, "BrokenLowerFont", &lowercase_garble), + test_text_item_with_font(2, "Helvetica", &healthy), + test_text_item_with_font(2, "BrokenUpperFont", &uppercase_garble), + ]; + + let quality = analyze_text_quality(&items); + + assert_eq!(quality.pages_needing_ocr, vec![1, 2]); + } + + #[test] + fn test_text_quality_cipher_stats_still_accumulate_across_fonts() { + let items: Vec = SHIFTED_CIPHER_TEXT + .split_whitespace() + .enumerate() + .map(|(index, chunk)| { + let font = match index % 4 { + 0 => "FixtureFontA", + 1 => "FixtureFontB", + 2 => "FixtureFontC", + _ => "FixtureFontD", + }; + test_text_item_with_font(1, font, chunk) + }) + .collect(); + + let quality = analyze_text_quality(&items); + + assert_eq!(quality.pages_needing_ocr, vec![1]); + } + #[test] fn test_text_quality_flags_localized_cid_mojibake_span() { let items = vec![ diff --git a/src/python.rs b/src/python.rs index 44141bd0..f1d3d376 100644 --- a/src/python.rs +++ b/src/python.rs @@ -808,8 +808,10 @@ fn detect_pdf_bytes(data: &[u8]) -> PyResult { Ok(to_py_result(result)) } -/// Lightweight PDF classification — returns type, page count, and OCR pages. -/// Faster than detect_pdf as it skips building the full PdfProcessResult. +/// Lightweight structural PDF classification — returns type, page count, and +/// pages with structural OCR signals. Faster than detect_pdf as it skips +/// building the full PdfProcessResult. It does not extract text or validate +/// text encoding quality. /// Pages in pages_needing_ocr are 0-indexed. #[pyfunction] fn classify_pdf(path: &str) -> PyResult { @@ -817,7 +819,8 @@ fn classify_pdf(path: &str) -> PyResult { classify_pdf_bytes(&data) } -/// Lightweight PDF classification from bytes. +/// Lightweight structural PDF classification from bytes. This fast path does +/// not extract text or validate text encoding quality. /// Pages in pages_needing_ocr are 0-indexed. #[pyfunction] fn classify_pdf_bytes(data: &[u8]) -> PyResult { diff --git a/src/text_quality.rs b/src/text_quality.rs index 04255b01..81bd5685 100644 --- a/src/text_quality.rs +++ b/src/text_quality.rs @@ -11,8 +11,9 @@ //! backstop on the region-extraction and whole-document paths. //! - **Item/span-level** ([`analyze_text_quality`], //! [`region_items_have_decoding_issue`]) run on individual `TextItem`s and -//! accumulate per-page evidence, so localized garbled spans on an otherwise -//! clean page are caught without a single span having to condemn the page. +//! accumulate page-wide and per-font evidence, so localized garbled spans on +//! an otherwise clean page are caught without a single span having to +//! condemn the page. //! //! Detection classes, roughly by signal: //! - **Replacement runs**: U+FFFD clusters ([`has_replacement_text_run`]). @@ -103,13 +104,47 @@ struct CipherGarbleStats { /// lowercase straight to uppercase mid-word. letter_bigrams: usize, case_shift_bigrams: usize, + /// Aggregate character composition and short word-like items/tokens. + non_whitespace_chars: usize, + explicit_word_tokens: usize, + explicit_word_letters: usize, + /// Streaming evidence for a syntactically valid Base64 payload. Whitespace + /// and PDF item boundaries are ignored so wrapped or fragmented payloads + /// are evaluated as a single aggregate sample. + base64_chars: usize, + base64_symbol_chars: usize, + base64_padding: usize, + base64_invalid: bool, + base64_saw_padding: bool, + base64_bit_buffer: u16, + base64_bit_count: u8, + base64_decoded_bytes: usize, + base64_printable_bytes: usize, } impl CipherGarbleStats { + const MIN_FONT_SAMPLE_LETTER_KINDS: usize = 15; + const MIN_FONT_SAMPLE_WORD_TOKENS: usize = 8; + const MIN_FONT_SAMPLE_WORD_LETTERS: usize = 100; + const MAX_WORD_TOKEN_CHARS: usize = 32; + fn add_text(&mut self, text: &str) { let mut prev: Option = None; + let mut token_chars = 0usize; + let mut token_letters = 0usize; for ch in text.chars() { + if ch.is_whitespace() { + self.add_explicit_word(token_chars, token_letters); + token_chars = 0; + token_letters = 0; + } else { + self.non_whitespace_chars += 1; + token_chars += 1; + self.add_base64_char(ch); + } + if ch.is_ascii_alphabetic() { + token_letters += 1; let idx = (ch.to_ascii_lowercase() as u8 - b'a') as usize; self.letter_counts[idx] += 1; self.ascii_letters += 1; @@ -134,6 +169,103 @@ impl CipherGarbleStats { prev = None; } } + + // PDF producers commonly emit each visible word as its own Tj/TJ + // item without a trailing space. Count a word-like trailing token as + // prose evidence; the aggregate Base64 check below prevents encoded + // chunks from becoming false word evidence. + self.add_explicit_word(token_chars, token_letters); + } + + fn add_explicit_word(&mut self, chars: usize, letters: usize) { + if (2..=Self::MAX_WORD_TOKEN_CHARS).contains(&chars) && letters * 2 >= chars { + self.explicit_word_tokens += 1; + self.explicit_word_letters += letters; + } + } + + fn has_explicit_prose_structure(&self) -> bool { + self.explicit_word_tokens >= Self::MIN_FONT_SAMPLE_WORD_TOKENS + && self.explicit_word_letters >= Self::MIN_FONT_SAMPLE_WORD_LETTERS + } + + fn add_base64_char(&mut self, ch: char) { + self.base64_chars += 1; + + if ch == '=' { + self.base64_symbol_chars += 1; + self.base64_padding += 1; + self.base64_saw_padding = true; + if self.base64_padding > 2 { + self.base64_invalid = true; + } + return; + } + + let value = match ch { + 'A'..='Z' => (ch as u8 - b'A') as u16, + 'a'..='z' => (ch as u8 - b'a' + 26) as u16, + '0'..='9' => (ch as u8 - b'0' + 52) as u16, + '+' => { + self.base64_symbol_chars += 1; + 62 + } + '/' => { + self.base64_symbol_chars += 1; + 63 + } + _ => { + self.base64_invalid = true; + return; + } + }; + + if self.base64_saw_padding { + self.base64_invalid = true; + return; + } + + self.base64_bit_buffer = (self.base64_bit_buffer << 6) | value; + self.base64_bit_count += 6; + if self.base64_bit_count >= 8 { + self.base64_bit_count -= 8; + let byte = (self.base64_bit_buffer >> self.base64_bit_count) as u8; + self.base64_decoded_bytes += 1; + if byte.is_ascii_graphic() || matches!(byte, b' ' | b'\t' | b'\n' | b'\r') { + self.base64_printable_bytes += 1; + } + let mask = (1u16 << self.base64_bit_count) - 1; + self.base64_bit_buffer &= mask; + } + } + + fn looks_like_structured_base64(&self) -> bool { + if self.base64_invalid + || self.base64_chars < 200 + || !self.base64_chars.is_multiple_of(4) + || self.base64_decoded_bytes == 0 + { + return false; + } + + let data_chars = self.base64_chars - self.base64_padding; + let padding_is_valid = match self.base64_padding { + 0 => data_chars.is_multiple_of(4) && self.base64_bit_count == 0, + 1 => data_chars % 4 == 3 && self.base64_bit_count == 2, + 2 => data_chars % 4 == 2 && self.base64_bit_count == 4, + _ => false, + }; + if !padding_is_valid || self.base64_bit_buffer != 0 { + return false; + } + + // `+`, `/`, or terminal padding is direct Base64 evidence. A Base64 + // encoding of plain text may contain none of those, so also accept a + // payload that decodes overwhelmingly to printable ASCII. Arbitrary + // digit-heavy cipher text can satisfy the alphabet and length rules, + // but its decoded bytes do not satisfy either discriminator. + self.base64_symbol_chars > 0 + || self.base64_printable_bytes * 10 >= self.base64_decoded_bytes * 9 } /// Cosine similarity between the observed letter histogram and English @@ -223,6 +355,38 @@ impl CipherGarbleStats { case_shifts || permuted_language } + + /// A per-font slice is more likely than a whole page to contain repeated + /// identifiers, acronyms, or other structured ASCII. A small alphabet can + /// accidentally resemble the sorted English frequency profile even when + /// the text is legitimate (for example, a dedicated code font containing + /// repeated `myXMLParser` labels). Require broad alphabet coverage before + /// applying the page-calibrated cipher heuristic to an isolated font. + /// Structured blobs such as Base64 can also cover the whole alphabet and + /// flip case frequently. Exempt only an aggregate that has valid Base64 + /// framing plus either Base64-specific symbols/padding or a predominantly + /// printable decoded payload. For other samples with a meaningful + /// digit/symbol share, require multiple short word-like tokens. This keeps + /// digit-heavy shifted tables in scope, including PDFs that emit one word + /// per item. Letter-only samples retain the original cipher behavior, + /// including uninterrupted shifted prose. + /// + /// This guard is intentionally font-only: page-wide detection keeps its + /// existing behavior, including all-lowercase and all-uppercase shifted + /// prose, while the reported mixed-font cipher sample covers well over + /// half of the ASCII alphabet. + fn font_sample_looks_garbled(&self) -> bool { + let letter_kinds = self + .letter_counts + .iter() + .filter(|&&count| count > 0) + .count(); + let letter_only = self.non_whitespace_chars == self.ascii_letters; + letter_kinds >= Self::MIN_FONT_SAMPLE_LETTER_KINDS + && !self.looks_like_structured_base64() + && (letter_only || self.has_explicit_prose_structure()) + && self.looks_garbled() + } } #[derive(Debug, Default)] @@ -239,6 +403,32 @@ struct PageTextQualityEvidence { replacement_spans: usize, longest_replacement_run: usize, cipher_garble: CipherGarbleStats, + cipher_garble_by_font: BTreeMap, +} + +impl PageTextQualityEvidence { + fn add_cipher_text(&mut self, font: &str, text: &str) { + self.cipher_garble.add_text(text); + if font.is_empty() { + return; + } + + if let Some(stats) = self.cipher_garble_by_font.get_mut(font) { + stats.add_text(text); + } else { + let mut stats = CipherGarbleStats::default(); + stats.add_text(text); + self.cipher_garble_by_font.insert(font.to_string(), stats); + } + } + + fn cipher_sample_looks_garbled(&self) -> bool { + self.cipher_garble.looks_garbled() + || self + .cipher_garble_by_font + .values() + .any(CipherGarbleStats::font_sample_looks_garbled) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -258,7 +448,7 @@ pub(crate) fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport { let evidence = evidence_by_page.entry(item.page).or_default(); evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count(); - evidence.cipher_garble.add_text(&item.text); + evidence.add_cipher_text(&item.font, &item.text); match text_span_decoding_issue_kind(&item.text) { Some(TextSpanIssueKind::Strong) => { @@ -282,7 +472,7 @@ pub(crate) fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport { if reasons_by_page.contains_key(&page) { continue; } - if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_garble.looks_garbled() + if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_sample_looks_garbled() { add_ocr_reason( &mut reasons_by_page,