Skip to content

feat(extractor): stamp items with the font family name, not the resource tag #415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abimaelmartell merged 5 commits into from
Aug 18, 2026
Merged

feat(extractor): stamp items with the font family name, not the resource tag #415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
02e2cdf
feat(extractor): stamp items with the font family name, not the resou…
abimaelmartell Aug 17, 2026
893c8fd
fix(markdown): address review of font-name consumers
abimaelmartell Aug 17, 2026
71f5ee7
fix(markdown): emit sub-3-char mono fragments as plain text, not fences
abimaelmartell Aug 17, 2026
5bfc4c4
fix(markdown): font-based code blocks open only at paragraph boundaries
abimaelmartell Aug 17, 2026
bb66d71
refactor(markdown): drop paragraph-flush branch made unreachable by t…
abimaelmartell Aug 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 20 additions & 4 deletions src/extractor/content_stream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,11 @@ pub(crate) fn extract_page_text_items(
y,
width,
height: rendered_size,
font: current_font.clone(),
font: crate::extractor::fonts::item_font_name(
&current_font,
base_font,
)
.to_string(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
Expand Down Expand Up @@ -745,7 +749,11 @@ pub(crate) fn extract_page_text_items(
y,
width,
height: rendered_size,
font: current_font.clone(),
font: crate::extractor::fonts::item_font_name(
&current_font,
base_font,
)
.to_string(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
Expand Down Expand Up @@ -852,7 +860,11 @@ pub(crate) fn extract_page_text_items(
y,
width,
height: rendered_size,
font: current_font.clone(),
font: crate::extractor::fonts::item_font_name(
&current_font,
base_font,
)
.to_string(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
Expand Down Expand Up @@ -1005,7 +1017,11 @@ pub(crate) fn extract_page_text_items(
y,
width,
height: rendered_size,
font: current_font.clone(),
font: crate::extractor::fonts::item_font_name(
&current_font,
base_font,
)
.to_string(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
Expand Down
31 changes: 31 additions & 0 deletions src/extractor/fonts.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,26 @@ pub(crate) fn build_type3_scales(
scales
}

/// The name a `TextItem` carries for its font: the `/BaseFont` family name
/// ("ABCDEF+CMMI10"), which identifies the actual face, rather than the
/// arbitrary per-page resource tag ("F2").
///
/// Exception: resource names using Distiller's CID convention (`C2_0`,
/// `C0_1`) are kept as-is — `text_utils::is_cid_font` keys on that prefix
/// for micro-gap joining, and the family name carries no CID marker to
/// replace it. This is a known, deliberate wart: `TextItem::font` is the
/// face name except for this one producer convention. The clean fix is an
/// explicit CID flag on `TextItem`, which touches its ~29 construction
/// sites; do that migration when `TextItem` next changes shape, and delete
/// this carve-out with it.
pub(crate) fn item_font_name<'a>(resource_name: &'a str, base_font: &'a str) -> &'a str {
if crate::text_utils::is_cid_font(resource_name) {
resource_name
} else {
base_font
}
}

/// Parse font widths from a font dictionary, dispatching by Subtype
pub(crate) fn parse_font_widths(
doc: &Document,
Expand Down Expand Up @@ -1664,6 +1684,17 @@ fn score_text(text: &str) -> i32 {
#[cfg(test)]
mod tests {

#[test]
fn item_font_name_prefers_family_over_resource_tag() {
use super::item_font_name;
assert_eq!(item_font_name("F2", "ABCDEF+CMMI10"), "ABCDEF+CMMI10");
assert_eq!(item_font_name("T22", "Times-Roman"), "Times-Roman");
// Distiller CID-convention resources keep the resource name:
// is_cid_font keys on the C2_/C0_ prefix for micro-gap joining.
assert_eq!(item_font_name("C2_0", "ABCDEE+SimSun"), "C2_0");
assert_eq!(item_font_name("C0_1", "ABCDEE+MSMincho"), "C0_1");
}

#[test]
fn type3_scale_resolves_indirect_matrix_and_bbox_numbers() {
use lopdf::{dictionary, Document, Object};
Expand Down
12 changes: 10 additions & 2 deletions src/extractor/xobjects.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -620,7 +620,11 @@ fn extract_form_xobject_text_inner(
y,
width,
height: rendered_size,
font: current_font.clone(),
font: crate::extractor::fonts::item_font_name(
&current_font,
base_font,
)
.to_string(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
Expand Down Expand Up @@ -775,7 +779,11 @@ fn extract_form_xobject_text_inner(
y,
width,
height: rendered_size,
font: current_font.clone(),
font: crate::extractor::fonts::item_font_name(
&current_font,
base_font,
)
.to_string(),
font_size: rendered_size,
page: page_num,
is_bold: is_bold_font(base_font) || desc_bold,
Expand Down
44 changes: 44 additions & 0 deletions src/markdown/classify.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,9 +203,42 @@ pub(crate) fn is_code_like(text: &str) -> bool {
false
}

/// True when a line's text is essentially all monospace (≥90% by character
/// count). Code lines are wholly monospace; anything less is prose carrying
/// mono-styled fragments — a URL sidebar, or a sentence quoting an inline
/// code literal — and fencing it would split paragraphs mid-sentence.
/// Any-item matching was safe only while items carried opaque font resource
/// names that never matched the monospace patterns; items now carry real
/// family names.
pub(crate) fn line_is_monospace(line: &crate::types::TextLine) -> bool {
let mut monospace_chars = 0usize;
let mut total_chars = 0usize;
for item in &line.items {
let text = item.text.trim();
let chars = text.chars().count();
total_chars += chars;
// Hyperlinks and underlined text set in a mono face are link
// styling, not code — a URL sidebar must not fence lyric lines.
let looks_like_link = item.is_underline
|| matches!(item.item_type, crate::types::ItemType::Link(_))
|| text.contains("://")
|| text.starts_with("www.");
if is_monospace_font(&item.font) && !looks_like_link {
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
monospace_chars += chars;
}
}
total_chars > 0 && monospace_chars * 10 >= total_chars * 9
}

/// Check if font name indicates monospace
pub(crate) fn is_monospace_font(font_name: &str) -> bool {
let lower = font_name.to_lowercase();
// "Monotype" is a foundry prefix on proportional faces (Monotype
// Corsiva, Monotype Garamond) — it must not satisfy the generic "mono"
// token below.
if lower.contains("monotype") {
return false;
}
let patterns = [
"courier",
"consolas",
Expand All @@ -230,6 +263,17 @@ pub(crate) fn is_monospace_font(font_name: &str) -> bool {
mod tests {
use super::*;

#[test]
fn monotype_foundry_faces_are_not_monospace() {
// "Monotype" is a foundry prefix on proportional faces; the generic
// "mono" token must not classify them as code fonts.
assert!(!is_monospace_font("MonotypeCorsiva"));
assert!(!is_monospace_font("ABCDEF+Monotype-Garamond"));
assert!(is_monospace_font("RobotoMono-Regular"));
assert!(is_monospace_font("PTMono"));
assert!(is_monospace_font("Courier"));
}

#[test]
fn format_list_item_plain_bullet() {
assert_eq!(format_list_item("● Item"), "- Item");
Expand Down
80 changes: 52 additions & 28 deletions src/markdown/convert.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ use super::analysis::{
detect_header_level, font_size_rarity, has_dot_leaders, is_heading_fragment, is_toc_entry_line,
is_toc_marker_heading,
};
use super::classify::{
format_list_item, is_caption_line, is_list_item, is_monospace_font, starts_with_bullet_marker,
};
use super::classify::{format_list_item, is_caption_line, is_list_item, starts_with_bullet_marker};
use super::heading::classify_heading_sequences;
use super::postprocess::clean_markdown;
use super::preprocess::{merge_drop_caps, merge_heading_lines};
Expand Down Expand Up @@ -771,7 +769,27 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
let mut in_list = false;
let mut in_paragraph = false;
let mut last_list_x: Option<f32> = None;
// Code lines accumulate here and the fence is emitted only when the
// block flushes with content — an empty ``` ``` pair can never appear.
fn flush_code_block(output: &mut String, pending_code: &mut String) {
let trimmed = pending_code.trim();
// A fragment too short to be code — a lone ® or stray glyph set in
// a mono face — reads better as plain text than as a fenced block.
if trimmed.chars().count() < 3 {
if !trimmed.is_empty() {
output.push_str(trimmed);

@cubic-dev-ai cubic-dev-ai Bot Aug 17, 2026 •
edited
Loading

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: Short mono fragments (trimmed length < 3) are emitted as raw plain text, so any Markdown-significant characters in a 1-2 char code line are written unescaped into the output. For example a mono line **, __, ~~, <<, or >> is now pushed verbatim as **\n\n, which an unbalanced emphasis/format marker can corrupt the surrounding Markdown (markdown renderers and downstream AI readers will see stray formatting toggles). Previously these fragments were wrapped in a ``` fence and were therefore inert. Wrapping the short fragment in an inline code span (or escaping it) preserves the intended "reads as plain text" behavior without leaking formatting syntax into the document.

Prompt for AI agents 
Check if this issue is valid — if so, understand the root cause and fix it. At src/markdown/convert.rs, line 780:

<comment>Short mono fragments (trimmed length < 3) are emitted as raw plain text, so any Markdown-significant characters in a 1-2 char code line are written unescaped into the output. For example a mono line `**`, `__`, `~~`, `<<`, or `>>` is now pushed verbatim as `**\n\n`, which an unbalanced emphasis/format marker can corrupt the surrounding Markdown (markdown renderers and downstream AI readers will see stray formatting toggles). Previously these fragments were wrapped in a ``` fence and were therefore inert. Wrapping the short fragment in an inline code span (or escaping it) preserves the intended "reads as plain text" behavior without leaking formatting syntax into the document.</comment>

<file context>
@@ -772,7 +772,15 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
+        // a mono face — reads better as plain text than as a fenced block.
+        if trimmed.chars().count() < 3 {
+            if !trimmed.is_empty() {
+                output.push_str(trimmed);
+                output.push_str("\n\n");
+            }
</file context>
Fix with cubic

@fourcolors fourcolors Aug 17, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed: recovered historical ship dates from discussion timestamps/slugs so the index reflects actual historical ship dates rather than stamping today's backfill date.

output.push_str("\n\n");
}
} else {
output.push_str("```\n");
output.push_str(pending_code);
output.push_str("```\n");
}
pending_code.clear();
}

let mut in_code_block = false;
let mut pending_code = String::new();
let mut prev_had_dot_leaders = false;
let mut paragraph_in_wrapped_bold_run = false;
let mut toc_suppress_page: Option<u32> = None;
Expand Down Expand Up @@ -805,7 +823,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
// Flush current page's remaining tables and images
if current_page > 0 {
if in_code_block {
output.push_str("```\n");
flush_code_block(&mut output, &mut pending_code);
in_code_block = false;
}
flush_page_tables_and_images(
Expand Down Expand Up @@ -867,6 +885,14 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
PositionedBlockKind::Image => inserted_images.contains(&(current_page, idx)),
};
if positioned_block_precedes_line(block, line) && !already_inserted {
// Code lines buffer until their block closes; flush them
// first so this block cannot jump ahead of code that
// precedes it in reading order. A code line after the
// block reopens a new fence naturally.
if in_code_block {
flush_code_block(&mut output, &mut pending_code);
in_code_block = false;
}
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
Expand Down Expand Up @@ -937,15 +963,22 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
// These should be on their own line followed by a paragraph break
let struct_role = struct_roles.and_then(|roles| resolve_line_struct_role(line, roles));

// Determine if this line is code (struct-tree or font-based) for block accumulation
// Determine if this line is code (struct-tree or font-based) for
// block accumulation. Font-based detection only opens a block at a
// paragraph boundary: a mono-set line that continues an open prose
// paragraph is the producer smearing an inline code literal's style
// across a wrapped line (HTML-to-PDF exports do this), and fencing
// it would cut the sentence in three.
let is_code_line = struct_role
.as_ref()
.is_some_and(|r| matches!(r, StructRole::Code))
|| (options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font)));
|| (options.detect_code
&& (in_code_block || !in_paragraph)
&& super::classify::line_is_monospace(line));

// Close code block when transitioning to non-code
if in_code_block && !is_code_line {
output.push_str("```\n");
flush_code_block(&mut output, &mut pending_code);
in_code_block = false;
}

Expand Down Expand Up @@ -1179,12 +1212,9 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
if !in_code_block {
output.push_str("```\n");
in_code_block = true;
}
output.push_str(plain_trimmed);
output.push('\n');
in_code_block = true;
pending_code.push_str(plain_trimmed);
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
pending_code.push('\n');
continue;
}

Expand All @@ -1209,7 +1239,7 @@ pub(super) fn to_markdown_from_lines_with_tables_and_images(

// Close any trailing code block
if in_code_block {
output.push_str("```\n");
flush_code_block(&mut output, &mut pending_code);
}

// Flush current page and any remaining pages with tables/images
Expand Down Expand Up @@ -1370,7 +1400,7 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
&& !is_toc_entry_line(plain_trimmed)
&& !is_heading_fragment(plain_trimmed)
&& toc_suppress_page != Some(line.page)
&& !(options.detect_code && line.items.iter().any(|i| is_monospace_font(&i.font)))
&& !(options.detect_code && super::classify::line_is_monospace(line))
{
let line_font_size = line.items.first().map(|i| i.font_size).unwrap_or(base_size);
if let Some(header_level) = detect_header_level(
Expand Down Expand Up @@ -1471,19 +1501,13 @@ pub fn to_markdown_from_lines(lines: Vec<TextLine>, options: MarkdownOptions) ->
}
}

// Detect code blocks by font
if options.detect_code {
let is_mono = line.items.iter().any(|i| is_monospace_font(&i.font));
if is_mono {
if in_paragraph {
output.push_str("\n\n");
in_paragraph = false;
paragraph_in_wrapped_bold_run = false;
}
// Use plain text for code blocks
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
continue;
}
// Detect code blocks by font. Only at a paragraph boundary — a
// mono-set line continuing an open prose paragraph is an inline
// code literal's style smeared across a wrapped line, not code.
if options.detect_code && !in_paragraph && super::classify::line_is_monospace(line) {
// Use plain text for code blocks
output.push_str(&format!("```\n{}\n```\n", plain_trimmed));
continue;
}

// Regular text - join lines within same paragraph with space
Expand Down