From 02e2cdf5e96f885ebc6c54483ca53c983fdb34da Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:18:57 -0700 Subject: [PATCH 1/5] feat(extractor): stamp items with the font family name, not the resource tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TextItem::font carried the page's font resource name ("F2", "T22") — an arbitrary per-page tag — even though both content-stream parsers already resolve the /BaseFont family name for bold/italic detection at every item-creation site. Stamp that resolved family name instead ("ABCDEF+CMMI10", "Courier"), from a single item_font_name helper so the two parsers cannot drift. One deliberate carve-out, documented on the helper: resource names using Distiller's CID convention (C2_0, C0_1) are kept as-is, because text_utils::is_cid_font keys on that prefix for micro-gap joining and the family name carries no CID marker to replace it. Consumers that match on font names start working against real names: - Code detection (is_monospace_font) previously never fired against opaque resource tags. It now does — so line classification also moves from any-item matching to a majority-by-characters rule (line_is_monospace): code lines are wholly monospace, while a lone URL or identifier styled in a mono face inside a prose line must not fence the surrounding sentence. - Heading/body font grouping now merges resource aliases of the same family instead of treating them as distinct fonts. - Positioned-item output (--items-json and the bindings) reports real face names. Regression corpus: code-heavy manuals improve substantially (assembly and C snippets previously emitted as prose now fence with line structure preserved); remaining churn reviewed as improvements. --- src/extractor/content_stream.rs | 24 +++++++++++++++++---- src/extractor/fonts.rs | 31 +++++++++++++++++++++++++++ src/extractor/xobjects.rs | 12 +++++++++-- src/markdown/classify.rs | 27 ++++++++++++++++++++++++ src/markdown/convert.rs | 37 ++++++++++++++++++++------------- 5 files changed, 110 insertions(+), 21 deletions(-) diff --git a/src/extractor/content_stream.rs b/src/extractor/content_stream.rs index ce688aa2..40c439f0 100644 --- a/src/extractor/content_stream.rs +++ b/src/extractor/content_stream.rs @@ -561,7 +561,11 @@ pub(crate) fn extract_page_text_items( y, width, height: rendered_size, - font: current_font.clone(), + font: crate::extractor::fonts::item_font_name( + ¤t_font, + base_font, + ) + .to_string(), font_size: rendered_size, page: page_num, is_bold: is_bold_font(base_font) || desc_bold, @@ -745,7 +749,11 @@ pub(crate) fn extract_page_text_items( y, width, height: rendered_size, - font: current_font.clone(), + font: crate::extractor::fonts::item_font_name( + ¤t_font, + base_font, + ) + .to_string(), font_size: rendered_size, page: page_num, is_bold: is_bold_font(base_font) || desc_bold, @@ -852,7 +860,11 @@ pub(crate) fn extract_page_text_items( y, width, height: rendered_size, - font: current_font.clone(), + font: crate::extractor::fonts::item_font_name( + ¤t_font, + base_font, + ) + .to_string(), font_size: rendered_size, page: page_num, is_bold: is_bold_font(base_font) || desc_bold, @@ -1005,7 +1017,11 @@ pub(crate) fn extract_page_text_items( y, width, height: rendered_size, - font: current_font.clone(), + font: crate::extractor::fonts::item_font_name( + ¤t_font, + base_font, + ) + .to_string(), font_size: rendered_size, page: page_num, is_bold: is_bold_font(base_font) || desc_bold, diff --git a/src/extractor/fonts.rs b/src/extractor/fonts.rs index dd5f374f..fec13264 100644 --- a/src/extractor/fonts.rs +++ b/src/extractor/fonts.rs @@ -225,6 +225,26 @@ pub(crate) fn build_type3_scales( scales } +/// The name a `TextItem` carries for its font: the `/BaseFont` family name +/// ("ABCDEF+CMMI10"), which identifies the actual face, rather than the +/// arbitrary per-page resource tag ("F2"). +/// +/// Exception: resource names using Distiller's CID convention (`C2_0`, +/// `C0_1`) are kept as-is — `text_utils::is_cid_font` keys on that prefix +/// for micro-gap joining, and the family name carries no CID marker to +/// replace it. This is a known, deliberate wart: `TextItem::font` is the +/// face name except for this one producer convention. The clean fix is an +/// explicit CID flag on `TextItem`, which touches its ~29 construction +/// sites; do that migration when `TextItem` next changes shape, and delete +/// this carve-out with it. +pub(crate) fn item_font_name<'a>(resource_name: &'a str, base_font: &'a str) -> &'a str { + if crate::text_utils::is_cid_font(resource_name) { + resource_name + } else { + base_font + } +} + /// Parse font widths from a font dictionary, dispatching by Subtype pub(crate) fn parse_font_widths( doc: &Document, @@ -1664,6 +1684,17 @@ fn score_text(text: &str) -> i32 { #[cfg(test)] mod tests { + #[test] + fn item_font_name_prefers_family_over_resource_tag() { + use super::item_font_name; + assert_eq!(item_font_name("F2", "ABCDEF+CMMI10"), "ABCDEF+CMMI10"); + assert_eq!(item_font_name("T22", "Times-Roman"), "Times-Roman"); + // Distiller CID-convention resources keep the resource name: + // is_cid_font keys on the C2_/C0_ prefix for micro-gap joining. + assert_eq!(item_font_name("C2_0", "ABCDEE+SimSun"), "C2_0"); + assert_eq!(item_font_name("C0_1", "ABCDEE+MSMincho"), "C0_1"); + } + #[test] fn type3_scale_resolves_indirect_matrix_and_bbox_numbers() { use lopdf::{dictionary, Document, Object}; diff --git a/src/extractor/xobjects.rs b/src/extractor/xobjects.rs index 5d52d211..66b3d7e2 100644 --- a/src/extractor/xobjects.rs +++ b/src/extractor/xobjects.rs @@ -620,7 +620,11 @@ fn extract_form_xobject_text_inner( y, width, height: rendered_size, - font: current_font.clone(), + font: crate::extractor::fonts::item_font_name( + ¤t_font, + base_font, + ) + .to_string(), font_size: rendered_size, page: page_num, is_bold: is_bold_font(base_font) || desc_bold, @@ -775,7 +779,11 @@ fn extract_form_xobject_text_inner( y, width, height: rendered_size, - font: current_font.clone(), + font: crate::extractor::fonts::item_font_name( + ¤t_font, + base_font, + ) + .to_string(), font_size: rendered_size, page: page_num, is_bold: is_bold_font(base_font) || desc_bold, diff --git a/src/markdown/classify.rs b/src/markdown/classify.rs index 868aa438..25fa20ec 100644 --- a/src/markdown/classify.rs +++ b/src/markdown/classify.rs @@ -203,6 +203,33 @@ pub(crate) fn is_code_like(text: &str) -> bool { false } +/// True when a line's text is essentially all monospace (≥90% by character +/// count). Code lines are wholly monospace; anything less is prose carrying +/// mono-styled fragments — a URL sidebar, or a sentence quoting an inline +/// code literal — and fencing it would split paragraphs mid-sentence. +/// Any-item matching was safe only while items carried opaque font resource +/// names that never matched the monospace patterns; items now carry real +/// family names. +pub(crate) fn line_is_monospace(line: &crate::types::TextLine) -> bool { + let mut monospace_chars = 0usize; + let mut total_chars = 0usize; + for item in &line.items { + let text = item.text.trim(); + let chars = text.chars().count(); + total_chars += chars; + // Hyperlinks and underlined text set in a mono face are link + // styling, not code — a URL sidebar must not fence lyric lines. + let looks_like_link = item.is_underline + || matches!(item.item_type, crate::types::ItemType::Link(_)) + || text.contains("://") + || text.starts_with("www."); + if is_monospace_font(&item.font) && !looks_like_link { + monospace_chars += chars; + } + } + total_chars > 0 && monospace_chars * 10 >= total_chars * 9 +} + /// Check if font name indicates monospace pub(crate) fn is_monospace_font(font_name: &str) -> bool { let lower = font_name.to_lowercase(); diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index 5e436c59..b15b7f6c 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -11,9 +11,7 @@ use super::analysis::{ detect_header_level, font_size_rarity, has_dot_leaders, is_heading_fragment, is_toc_entry_line, is_toc_marker_heading, }; -use super::classify::{ - format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker, -}; +use super::classify::{format_list_item, is_caption_line, is_list_item, starts_with_bullet_marker}; use super::heading::classify_heading_sequences; use super::postprocess::clean_markdown; use super::preprocess::{merge_drop_caps, merge_heading_lines}; @@ -771,7 +769,19 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( let mut in_list = false; let mut in_paragraph = false; let mut last_list_x: Option = None; + // Code lines accumulate here and the fence is emitted only when the + // block flushes with content — an empty ``` ``` pair can never appear. + fn flush_code_block(output: &mut String, pending_code: &mut String) { + if !pending_code.trim().is_empty() { + output.push_str("```\n"); + output.push_str(pending_code); + output.push_str("```\n"); + } + pending_code.clear(); + } + let mut in_code_block = false; + let mut pending_code = String::new(); let mut prev_had_dot_leaders = false; let mut paragraph_in_wrapped_bold_run = false; let mut toc_suppress_page: Option = None; @@ -805,7 +815,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Flush current page's remaining tables and images if current_page > 0 { if in_code_block { - output.push_str("```\n"); + flush_code_block(&mut output, &mut pending_code); in_code_block = false; } flush_page_tables_and_images( @@ -941,11 +951,11 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( let is_code_line = struct_role .as_ref() .is_some_and(|r| matches!(r, StructRole::Code)) - || (options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font))); + || (options.detect_code && super::classify::line_is_monospace(line)); // Close code block when transitioning to non-code if in_code_block && !is_code_line { - output.push_str("```\n"); + flush_code_block(&mut output, &mut pending_code); in_code_block = false; } @@ -1179,12 +1189,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( in_paragraph = false; paragraph_in_wrapped_bold_run = false; } - if !in_code_block { - output.push_str("```\n"); - in_code_block = true; - } - output.push_str(plain_trimmed); - output.push('\n'); + in_code_block = true; + pending_code.push_str(plain_trimmed); + pending_code.push('\n'); continue; } @@ -1209,7 +1216,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Close any trailing code block if in_code_block { - output.push_str("```\n"); + flush_code_block(&mut output, &mut pending_code); } // Flush current page and any remaining pages with tables/images @@ -1370,7 +1377,7 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> && !is_toc_entry_line(plain_trimmed) && !is_heading_fragment(plain_trimmed) && toc_suppress_page != Some(line.page) - && !(options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font))) + && !(options.detect_code && super::classify::line_is_monospace(line)) { let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size); if let Some(header_level) = detect_header_level( @@ -1473,7 +1480,7 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> // Detect code blocks by font if options.detect_code { - let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font)); + let is_mono = super::classify::line_is_monospace(line); if is_mono { if in_paragraph { output.push_str("\n\n"); From 893c8fd44da16a496ad2e12bd4eb2ca5bc2a0eaa Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:12:48 -0700 Subject: [PATCH 2/5] fix(markdown): address review of font-name consumers - Monotype is a foundry prefix on proportional faces (Monotype Corsiva, Monotype Garamond); it must not satisfy is_monospace_font's generic "mono" token. Regression tests pin both directions. - Flush the pending code block before inserting a positioned table or image, so a block that falls between two code lines cannot be emitted ahead of code that precedes it in reading order; a code line after the block reopens a new fence naturally. --- src/markdown/classify.rs | 17 +++++++++++++++++ src/markdown/convert.rs | 8 ++++++++ 2 files changed, 25 insertions(+) diff --git a/src/markdown/classify.rs b/src/markdown/classify.rs index 25fa20ec..0a828f5e 100644 --- a/src/markdown/classify.rs +++ b/src/markdown/classify.rs @@ -233,6 +233,12 @@ pub(crate) fn line_is_monospace(line: &crate::types::TextLine) -> bool { /// Check if font name indicates monospace pub(crate) fn is_monospace_font(font_name: &str) -> bool { let lower = font_name.to_lowercase(); + // "Monotype" is a foundry prefix on proportional faces (Monotype + // Corsiva, Monotype Garamond) — it must not satisfy the generic "mono" + // token below. + if lower.contains("monotype") { + return false; + } let patterns = [ "courier", "consolas", @@ -257,6 +263,17 @@ pub(crate) fn is_monospace_font(font_name: &str) -> bool { mod tests { use super::*; + #[test] + fn monotype_foundry_faces_are_not_monospace() { + // "Monotype" is a foundry prefix on proportional faces; the generic + // "mono" token must not classify them as code fonts. + assert!(!is_monospace_font("MonotypeCorsiva")); + assert!(!is_monospace_font("ABCDEF+Monotype-Garamond")); + assert!(is_monospace_font("RobotoMono-Regular")); + assert!(is_monospace_font("PTMono")); + assert!(is_monospace_font("Courier")); + } + #[test] fn format_list_item_plain_bullet() { assert_eq!(format_list_item("● Item"), "- Item"); diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index b15b7f6c..822b20e4 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -877,6 +877,14 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( PositionedBlockKind::Image => inserted_images.contains(&(current_page, idx)), }; if positioned_block_precedes_line(block, line) && !already_inserted { + // Code lines buffer until their block closes; flush them + // first so this block cannot jump ahead of code that + // precedes it in reading order. A code line after the + // block reopens a new fence naturally. + if in_code_block { + flush_code_block(&mut output, &mut pending_code); + in_code_block = false; + } if in_paragraph { output.push_str("\n\n"); in_paragraph = false; From 71f5ee79e0ea2bbd900281fbb6a0dc4a018fd767 Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:28:08 -0700 Subject: [PATCH 3/5] fix(markdown): emit sub-3-char mono fragments as plain text, not fences A lone registered-trademark glyph or stray bullet set in a mono face is not code; a fenced block containing one character reads as noise. --- src/markdown/convert.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index 822b20e4..ba13f66a 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -772,7 +772,15 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // Code lines accumulate here and the fence is emitted only when the // block flushes with content — an empty ``` ``` pair can never appear. fn flush_code_block(output: &mut String, pending_code: &mut String) { - if !pending_code.trim().is_empty() { + let trimmed = pending_code.trim(); + // A fragment too short to be code — a lone ® or stray glyph set in + // a mono face — reads better as plain text than as a fenced block. + if trimmed.chars().count() < 3 { + if !trimmed.is_empty() { + output.push_str(trimmed); + output.push_str("\n\n"); + } + } else { output.push_str("```\n"); output.push_str(pending_code); output.push_str("```\n"); From 5bfc4c4a92c0c9755bfce7a145e58fda25e23518 Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:37:09 -0700 Subject: [PATCH 4/5] fix(markdown): font-based code blocks open only at paragraph boundaries HTML-to-PDF producers smear an inline code literal's mono style across whole wrapped lines, so a prose paragraph can alternate body and mono fonts line by line. Fencing those lines cut sentences in three: prose head, fenced middle, prose tail. A mono-set line that continues an open prose paragraph now stays prose; font-based blocks open at paragraph boundaries (or continue an open block), and struct-tree Code roles are honored unconditionally. --- src/markdown/convert.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index ba13f66a..16b4da11 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -963,11 +963,18 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images( // These should be on their own line followed by a paragraph break let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(line, roles)); - // Determine if this line is code (struct-tree or font-based) for block accumulation + // Determine if this line is code (struct-tree or font-based) for + // block accumulation. Font-based detection only opens a block at a + // paragraph boundary: a mono-set line that continues an open prose + // paragraph is the producer smearing an inline code literal's style + // across a wrapped line (HTML-to-PDF exports do this), and fencing + // it would cut the sentence in three. let is_code_line = struct_role .as_ref() .is_some_and(|r| matches!(r, StructRole::Code)) - || (options.detect_code && super::classify::line_is_monospace(line)); + || (options.detect_code + && (in_code_block || !in_paragraph) + && super::classify::line_is_monospace(line)); // Close code block when transitioning to non-code if in_code_block && !is_code_line { @@ -1494,8 +1501,10 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> } } - // Detect code blocks by font - if options.detect_code { + // Detect code blocks by font. Only at a paragraph boundary — a + // mono-set line continuing an open prose paragraph is an inline + // code literal's style smeared across a wrapped line, not code. + if options.detect_code && !in_paragraph { let is_mono = super::classify::line_is_monospace(line); if is_mono { if in_paragraph { From bb66d7175e0d0b4fe8aa0e4356c59d178234f39c Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:46:59 -0700 Subject: [PATCH 5/5] refactor(markdown): drop paragraph-flush branch made unreachable by the boundary gate The enclosing guard proves in_paragraph is false, so the nested flush could never run; the guard and mono check collapse into one condition. --- src/markdown/convert.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/markdown/convert.rs b/src/markdown/convert.rs index 16b4da11..00f00231 100644 --- a/src/markdown/convert.rs +++ b/src/markdown/convert.rs @@ -1504,18 +1504,10 @@ pub fn to_markdown_from_lines(lines: Vec, options: MarkdownOptions) -> // Detect code blocks by font. Only at a paragraph boundary — a // mono-set line continuing an open prose paragraph is an inline // code literal's style smeared across a wrapped line, not code. - if options.detect_code && !in_paragraph { - let is_mono = super::classify::line_is_monospace(line); - if is_mono { - if in_paragraph { - output.push_str("\n\n"); - in_paragraph = false; - paragraph_in_wrapped_bold_run = false; - } - // Use plain text for code blocks - output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); - continue; - } + if options.detect_code && !in_paragraph && super::classify::line_is_monospace(line) { + // Use plain text for code blocks + output.push_str(&format!("```\n{}\n```\n", plain_trimmed)); + continue; } // Regular text - join lines within same paragraph with space