Skip to content
93 changes: 93 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6259,6 +6259,13 @@ mod tests {
}
}

fn test_text_item_with_font(page: u32, font: &str, text: &str) -> TextItem {
TextItem {
font: font.to_string(),
..test_text_item_on_page(page, text)
}
}

#[test]
fn removed_sparse_folios_leave_no_layout_evidence() {
let items = vec![
Expand Down Expand Up @@ -6581,6 +6588,92 @@ mod tests {
assert_eq!(quality.pages_needing_ocr, vec![1]);
}

#[test]
fn test_text_quality_flags_garbled_font_on_mixed_page() {
let healthy = CAESAR_PROSE.repeat(3);
let items = vec![
test_text_item_with_font(1, "Helvetica", &healthy),
test_text_item_with_font(1, "FixtureFont", SHIFTED_CIPHER_TEXT),
];

let quality = analyze_text_quality(&items);

assert_eq!(quality.pages_needing_ocr, vec![1]);
assert_eq!(
quality.reasons_by_page.get(&1).cloned(),
Some(vec![OCR_REASON_SUSPECTED_GARBLED_TEXT.to_string()])
);
}

#[test]
fn test_text_quality_allows_clean_multi_font_page() {
let items = vec![
test_text_item_with_font(1, "Helvetica", CAESAR_PROSE),
test_text_item_with_font(1, "TimesRoman", &CAESAR_PROSE.repeat(2)),
];

let quality = analyze_text_quality(&items);

assert!(!quality.has_encoding_issues);
assert!(quality.pages_needing_ocr.is_empty());
}

#[test]
fn test_text_quality_allows_structured_ascii_in_dedicated_fonts() {
let healthy = CAESAR_PROSE.repeat(3);
let mixed_case_code = "myXMLParser ".repeat(20);
let uppercase_code = "MYXMLPARSER ".repeat(20);
let items = vec![
test_text_item_with_font(1, "Helvetica", &healthy),
test_text_item_with_font(1, "CodeFont", &mixed_case_code),
test_text_item_with_font(2, "Helvetica", &healthy),
test_text_item_with_font(2, "AcronymFont", &uppercase_code),
];

let quality = analyze_text_quality(&items);

assert!(!quality.has_encoding_issues);
assert!(quality.pages_needing_ocr.is_empty());
}

#[test]
fn test_text_quality_flags_uniform_case_garbled_fonts_on_mixed_pages() {
let healthy = CAESAR_PROSE.repeat(3);
let lowercase_garble = caesar_shift(&CAESAR_PROSE.to_lowercase(), 5);
let uppercase_garble = caesar_shift(&CAESAR_PROSE.to_uppercase(), 7);
let items = vec![
test_text_item_with_font(1, "Helvetica", &healthy),
test_text_item_with_font(1, "BrokenLowerFont", &lowercase_garble),
test_text_item_with_font(2, "Helvetica", &healthy),
test_text_item_with_font(2, "BrokenUpperFont", &uppercase_garble),
];

let quality = analyze_text_quality(&items);

assert_eq!(quality.pages_needing_ocr, vec![1, 2]);
}

#[test]
fn test_text_quality_cipher_stats_still_accumulate_across_fonts() {
let items: Vec<TextItem> = SHIFTED_CIPHER_TEXT
.split_whitespace()
.enumerate()
.map(|(index, chunk)| {
let font = match index % 4 {
0 => "FixtureFontA",
1 => "FixtureFontB",
2 => "FixtureFontC",
_ => "FixtureFontD",
};
test_text_item_with_font(1, font, chunk)
})
.collect();

let quality = analyze_text_quality(&items);

assert_eq!(quality.pages_needing_ocr, vec![1]);
}

#[test]
fn test_text_quality_flags_localized_cid_mojibake_span() {
let items = vec![
Expand Down
57 changes: 53 additions & 4 deletions src/text_quality.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@
//! backstop on the region-extraction and whole-document paths.
//! - **Item/span-level** ([`analyze_text_quality`],
//! [`region_items_have_decoding_issue`]) run on individual `TextItem`s and
//! accumulate per-page evidence, so localized garbled spans on an otherwise
//! clean page are caught without a single span having to condemn the page.
//! accumulate page-wide and per-font evidence, so localized garbled spans on
//! an otherwise clean page are caught without a single span having to
//! condemn the page.
//!
//! Detection classes, roughly by signal:
//! - **Replacement runs**: U+FFFD clusters ([`has_replacement_text_run`]).
Expand Down Expand Up @@ -106,6 +107,8 @@ struct CipherGarbleStats {
}

impl CipherGarbleStats {
const MIN_FONT_SAMPLE_LETTER_KINDS: usize = 15;

fn add_text(&mut self, text: &str) {
let mut prev: Option<char> = None;
for ch in text.chars() {
Expand Down Expand Up @@ -223,6 +226,26 @@ impl CipherGarbleStats {

case_shifts || permuted_language
}

/// A per-font slice is more likely than a whole page to contain repeated
/// identifiers, acronyms, or other structured ASCII. A small alphabet can
/// accidentally resemble the sorted English frequency profile even when
/// the text is legitimate (for example, a dedicated code font containing
/// repeated `myXMLParser` labels). Require broad alphabet coverage before
/// applying the page-calibrated cipher heuristic to an isolated font.
///
/// This guard is intentionally font-only: page-wide detection keeps its
/// existing behavior, including all-lowercase and all-uppercase shifted
/// prose, while the reported mixed-font cipher sample covers well over
/// half of the ASCII alphabet.
fn font_sample_looks_garbled(&self) -> bool {
let letter_kinds = self
.letter_counts
.iter()
.filter(|&&count| count > 0)
.count();
letter_kinds >= Self::MIN_FONT_SAMPLE_LETTER_KINDS && self.looks_garbled()
Comment thread
AnnaSuSu marked this conversation as resolved.
Outdated
}
}

#[derive(Debug, Default)]
Expand All @@ -239,6 +262,32 @@ struct PageTextQualityEvidence {
replacement_spans: usize,
longest_replacement_run: usize,
cipher_garble: CipherGarbleStats,
cipher_garble_by_font: BTreeMap<String, CipherGarbleStats>,
}

impl PageTextQualityEvidence {
fn add_cipher_text(&mut self, font: &str, text: &str) {
self.cipher_garble.add_text(text);
if font.is_empty() {
return;
}

if let Some(stats) = self.cipher_garble_by_font.get_mut(font) {
stats.add_text(text);
} else {
let mut stats = CipherGarbleStats::default();
stats.add_text(text);
self.cipher_garble_by_font.insert(font.to_string(), stats);
}
}

fn cipher_sample_looks_garbled(&self) -> bool {
self.cipher_garble.looks_garbled()
|| self
.cipher_garble_by_font
.values()
.any(CipherGarbleStats::font_sample_looks_garbled)
}
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
Expand All @@ -258,7 +307,7 @@ pub(crate) fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {

let evidence = evidence_by_page.entry(item.page).or_default();
evidence.chars += item.text.chars().filter(|ch| !ch.is_whitespace()).count();
evidence.cipher_garble.add_text(&item.text);
evidence.add_cipher_text(&item.font, &item.text);

match text_span_decoding_issue_kind(&item.text) {
Some(TextSpanIssueKind::Strong) => {
Expand All @@ -282,7 +331,7 @@ pub(crate) fn analyze_text_quality(items: &[TextItem]) -> TextQualityReport {
if reasons_by_page.contains_key(&page) {
continue;
}
if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_garble.looks_garbled()
if page_replacement_evidence_needs_ocr(&evidence) || evidence.cipher_sample_looks_garbled()
{
add_ocr_reason(
&mut reasons_by_page,
Expand Down