diff --git a/src/tables/detect_lines.rs b/src/tables/detect_lines.rs index 492d47c8..bdafb9f9 100644 --- a/src/tables/detect_lines.rs +++ b/src/tables/detect_lines.rs @@ -9,7 +9,7 @@ use std::collections::{HashMap, HashSet}; use crate::tables::Table; use crate::types::{PdfLine, PdfRect, TextItem}; -use super::detect_rects::{assign_items_to_grid, snap_edges}; +use super::detect_rects::{assign_items_to_ruled_grid, snap_edges}; const RULE_Y_TOLERANCE: f32 = 2.0; const RULE_JOIN_GAP: f32 = 6.0; @@ -956,7 +956,8 @@ fn build_open_edge_grid_table_for_rules( return None; } - let (body_cells, mut item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); + let (body_cells, mut item_indices) = + assign_items_to_ruled_grid(items, &col_edges, &row_edges, page); let column_count = col_edges.len() - 1; let occupied_body_rows = body_cells .iter() @@ -1772,7 +1773,8 @@ fn detect_tables_from_lines_inner( ); // Assign items to grid - let (cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges_desc, page); + let (cells, item_indices) = + assign_items_to_ruled_grid(items, &col_edges, &row_edges_desc, page); // Require at least 2 non-empty rows let non_empty_rows = cells diff --git a/src/tables/detect_rects.rs b/src/tables/detect_rects.rs index 130cdf70..eaf80aaa 100644 --- a/src/tables/detect_rects.rs +++ b/src/tables/detect_rects.rs @@ -6,6 +6,7 @@ use log::debug; use crate::types::{PdfRect, TextItem}; +use super::financial::try_split_financial_item_across_rule; use super::Table; const DOMINANT_PAGE_BACKGROUND_MIN_REPETITIONS: usize = 8; @@ -1530,7 +1531,7 @@ fn try_build_grid( } // Build table: assign text items to cells - let (mut cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); + let (mut cells, item_indices) = assign_items_to_ruled_grid(items, &col_edges, &row_edges, page); // Consolidate vertically-merged cells: rects spanning multiple grid rows // should have their text collected into the first sub-row. @@ -1682,6 +1683,25 @@ pub(crate) fn assign_items_to_grid( col_edges: &[f32], row_edges: &[f32], page: u32, +) -> (Vec>, Vec) { + assign_items_to_grid_with_options(items, col_edges, row_edges, page, false) +} + +pub(crate) fn assign_items_to_ruled_grid( + items: &[TextItem], + col_edges: &[f32], + row_edges: &[f32], + page: u32, +) -> (Vec>, Vec) { + assign_items_to_grid_with_options(items, col_edges, row_edges, page, true) +} + +fn assign_items_to_grid_with_options( + items: &[TextItem], + col_edges: &[f32], + row_edges: &[f32], + page: u32, + allow_financial_splits: bool, ) -> (Vec>, Vec) { let num_cols = col_edges.len() - 1; let num_rows = row_edges.len() - 1; @@ -1691,25 +1711,94 @@ pub(crate) fn assign_items_to_grid( vec![vec![Vec::new(); num_cols]; num_rows]; let mut indices = Vec::new(); + // A TJ array can join two table values into one item when their gap is + // word-sized but smaller than the extractor's column-gap threshold. Split + // pure numeric items only when their geometry crosses a ruled boundary; + // values contained by one cell will be joined back together below. + let mut split_items = Vec::new(); + let mut split_ranges = vec![None; items.len()]; for (idx, item) in items.iter().enumerate() { - if item.page != page { + if item.page != page || item.width <= 0.0 { continue; } - // Use item center for assignment - let cx = item.x + item.width / 2.0; - let cy = item.y; + let right = item.x + item.width; + let crosses_rule = allow_financial_splits + && col_edges + .get(1..col_edges.len().saturating_sub(1)) + .is_some_and(|inner_edges| { + inner_edges + .iter() + .any(|&edge| item.x < edge && right > edge) + }); + if !crosses_rule { + continue; + } + if let Some(sub_items) = try_split_financial_item_across_rule(item) { + let start = split_items.len(); + split_items.extend(sub_items); + debug!( + "split financial item across ruled column boundary: x={:.2} width={:.2} edge_count={} sub_count={}", + item.x, + item.width, + col_edges.len(), + split_items.len() - start + ); + split_ranges[idx] = Some(start..split_items.len()); + } + } - // Find column: cx must be between col_edges[c] and col_edges[c+1] - let col = (0..num_cols).find(|&c| cx >= col_edges[c] - 2.0 && cx <= col_edges[c + 1] + 2.0); + for (idx, item) in items.iter().enumerate() { + if item.page != page { + continue; + } // Find row: cy must be between row_edges[r+1] (bottom) and row_edges[r] (top) - let row = (0..num_rows).find(|&r| cy >= row_edges[r + 1] - 2.0 && cy <= row_edges[r] + 2.0); + let row = (0..num_rows) + .find(|&r| item.y >= row_edges[r + 1] - 2.0 && item.y <= row_edges[r] + 2.0); - if let (Some(c), Some(r)) = (col, row) { - cell_items[r][c].push((idx, item)); - indices.push(idx); + if let Some(r) = row { + if let Some(range) = split_ranges[idx].clone() { + let candidates: Vec<&TextItem> = split_items[range].iter().collect(); + let placements: Vec> = candidates + .iter() + .map(|candidate| { + let cx = candidate.x + candidate.width / 2.0; + (0..num_cols) + .find(|&c| cx >= col_edges[c] - 2.0 && cx <= col_edges[c + 1] + 2.0) + }) + .collect(); + + if placements.iter().all(Option::is_some) { + for (candidate, col) in candidates.into_iter().zip(placements) { + let Some(c) = col else { continue }; + cell_items[r][c].push((idx, candidate)); + } + indices.push(idx); + } else { + // A split value outside every column is a bad split. Keep + // the original item together so no value is silently removed. + let cx = item.x + item.width / 2.0; + if let Some(c) = (0..num_cols) + .find(|&c| cx >= col_edges[c] - 2.0 && cx <= col_edges[c + 1] + 2.0) + { + cell_items[r][c].push((idx, item)); + indices.push(idx); + } + } + } else { + let cx = item.x + item.width / 2.0; + if let Some(c) = (0..num_cols) + .find(|&c| cx >= col_edges[c] - 2.0 && cx <= col_edges[c + 1] + 2.0) + { + cell_items[r][c].push((idx, item)); + indices.push(idx); + } + } } } + indices.sort_unstable(); + indices.dedup(); + // Build cell strings: sort items within each cell by Y descending then X ascending let mut cells: Vec> = Vec::with_capacity(num_rows); for row_items in &mut cell_items { @@ -2828,7 +2917,7 @@ fn detect_row_stripe_table_from_cell_rects( page_items.len() ); - let (mut cells, item_indices) = assign_items_to_grid(items, &col_edges, &row_edges, page); + let (mut cells, item_indices) = assign_items_to_ruled_grid(items, &col_edges, &row_edges, page); if item_indices.is_empty() { return None; @@ -3457,6 +3546,39 @@ mod tests { } } + #[test] + fn consolidated_numeric_item_crossing_rule_boundary_splits_cells() { + let mut item = make_item("236,480,212 10,024,724", 367.05, 524.45, 11.0); + item.width = 115.35; + + let (cells, indices) = assign_items_to_ruled_grid( + &[item], + &[300.0, 356.4, 428.35, 485.85, 600.0], + &[600.0, 500.0, 400.0], + 1, + ); + + assert_eq!(cells[0][1], "236,480,212", "cells: {cells:?}"); + assert_eq!(cells[0][2], "10,024,724", "cells: {cells:?}"); + assert_eq!(indices, vec![0]); + } + + #[test] + fn consolidated_numeric_item_stays_together_in_inferred_grids() { + let mut item = make_item("236,480,212 10,024,724", 367.05, 524.45, 11.0); + item.width = 115.35; + + let (cells, indices) = assign_items_to_grid( + &[item], + &[300.0, 356.4, 428.35, 485.85, 600.0], + &[600.0, 500.0, 400.0], + 1, + ); + + assert_eq!(cells[0][1], "236,480,212 10,024,724", "cells: {cells:?}"); + assert_eq!(indices, vec![0]); + } + // --- is_chart_bar_cluster / detect_chart_regions --- /// Stacked bar chart: frame + 3 columns of equal-width segments with diff --git a/src/tables/financial.rs b/src/tables/financial.rs index 70d102be..1c0f1461 100644 --- a/src/tables/financial.rs +++ b/src/tables/financial.rs @@ -1,5 +1,6 @@ //! Financial token splitting for consolidated value items. +use crate::types::ItemType; use crate::types::TextItem; /// Check if a whitespace-separated token looks like a financial number. @@ -77,11 +78,16 @@ pub(crate) fn tokenize_financial_values(text: &str) -> Option> { } } -/// Try to split a consolidated financial item into individual sub-items. -/// Criteria: width > font_size × 20, no alphabetic words, tokenization yields 3+ values. -/// Creates sub-items with evenly-distributed X positions across the original item's span. -pub(crate) fn try_split_financial_item(item: &TextItem) -> Option> { - if item.width <= item.font_size * 20.0 { +fn try_split_financial_item_with_limits( + item: &TextItem, + width_font_multiple: f32, + min_values: usize, + center_in_slot: bool, +) -> Option> { + if item.font_size <= 0.0 || !matches!(item.item_type, ItemType::Text) { + return None; + } + if item.width <= item.font_size * width_font_multiple { return None; } let text = &item.text; @@ -89,7 +95,7 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option> return None; } let values = tokenize_financial_values(text)?; - if values.len() < 3 { + if values.len() < min_values { return None; } let n = values.len() as f32; @@ -97,9 +103,10 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option> let sub_width = spacing * 0.9; let mut sub_items = Vec::with_capacity(values.len()); for (i, val) in values.iter().enumerate() { + let slot_offset = if center_in_slot { spacing * 0.5 } else { 0.0 }; sub_items.push(TextItem { text: val.clone(), - x: item.x + spacing * i as f32 + spacing * 0.5, + x: item.x + spacing * i as f32 + slot_offset, y: item.y, width: sub_width, height: item.height, @@ -116,3 +123,83 @@ pub(crate) fn try_split_financial_item(item: &TextItem) -> Option> } Some(sub_items) } + +/// Try to split a consolidated financial item into individual sub-items. +/// Criteria: width > font_size × 20, no alphabetic words, tokenization yields 3+ values. +/// Creates sub-items with evenly-distributed X positions across the original item's span. +pub(crate) fn try_split_financial_item(item: &TextItem) -> Option> { + try_split_financial_item_with_limits(item, 20.0, 3, true) +} + +/// Split a two-value item that physically crosses a ruled column boundary. +/// +/// Ruled geometry supplies stronger evidence than the heuristic-table width +/// heuristic, so a word-space join across two cells is allowed to split even +/// when it contains only two values. +pub(crate) fn try_split_financial_item_across_rule(item: &TextItem) -> Option> { + try_split_financial_item_with_limits(item, 8.0, 2, false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::ItemType; + + fn numeric_pair_item() -> TextItem { + TextItem { + text: "236,480,212 10,024,724".to_string(), + x: 367.0, + y: 524.0, + width: 115.0, + height: 11.0, + font: "TestFont".to_string(), + font_size: 11.0, + page: 1, + is_bold: false, + is_italic: false, + is_underline: false, + is_strikeout: false, + item_type: ItemType::Text, + mcid: None, + } + } + + #[test] + fn heuristic_splitter_keeps_two_value_items_intact() { + assert!(try_split_financial_item(&numeric_pair_item()).is_none()); + } + + #[test] + fn ruled_boundary_splitter_splits_two_value_items() { + let split = try_split_financial_item_across_rule(&numeric_pair_item()) + .expect("a wide two-value item crossing a rule should split"); + assert_eq!(split.len(), 2); + assert_eq!(split[0].text, "236,480,212"); + assert_eq!(split[1].text, "10,024,724"); + } + + #[test] + fn ruled_boundary_splitter_rejects_form_fields() { + let mut item = numeric_pair_item(); + item.font_size = 0.0; + assert!(try_split_financial_item_across_rule(&item).is_none()); + + item.font_size = 11.0; + item.item_type = ItemType::FormField; + assert!(try_split_financial_item_across_rule(&item).is_none()); + } + + #[test] + fn heuristic_and_ruled_splitters_use_their_original_slot_alignment() { + let mut heuristic_item = numeric_pair_item(); + heuristic_item.text = "1 2 3".to_string(); + heuristic_item.width = 230.0; + let heuristic_split = try_split_financial_item(&heuristic_item) + .expect("three wide values should use the heuristic splitter"); + assert_eq!(heuristic_split[0].x, heuristic_item.x + 230.0 / 6.0); + + let ruled_split = try_split_financial_item_across_rule(&numeric_pair_item()) + .expect("a wide two-value item crossing a rule should split"); + assert_eq!(ruled_split[0].x, numeric_pair_item().x); + } +} diff --git a/src/tables/grid.rs b/src/tables/grid.rs index de4356bf..f2501b76 100644 --- a/src/tables/grid.rs +++ b/src/tables/grid.rs @@ -179,52 +179,79 @@ fn merge_numeric_adjacent_clusters( items: &[(usize, &TextItem)], threshold: f32, ) -> Vec> { - // For each cluster, compute: center, item count, numeric fraction + use std::collections::{HashMap, VecDeque}; + + // For each cluster, compute: center, item count, numeric fraction. + // Classification consumes the exact item X values belonging to each + // cluster; jitter near another center must not reclassify a cluster. struct ClusterInfo { center: f32, count: usize, numeric_frac: f32, } - let compute_info = |xs: &[f32]| -> ClusterInfo { - let center = xs.iter().sum::() / xs.len() as f32; - // Count items and numeric fraction for items near this cluster center - let mut total = 0; - let mut numeric = 0; - for (_, item) in items { - if (item.x - center).abs() < threshold { - total += 1; - if is_numeric_text(&item.text) { - numeric += 1; + let mut numeric_flags_by_x: HashMap> = HashMap::new(); + for (_, item) in items { + numeric_flags_by_x + .entry(item.x.to_bits()) + .or_default() + .push_back(is_numeric_text(&item.text)); + } + + let compute_infos = |numeric_flags_by_x: &mut HashMap>, + clusters: &[Vec]| + -> Vec { + let centers: Vec = clusters + .iter() + .map(|xs| xs.iter().sum::() / xs.len() as f32) + .collect(); + let mut totals = vec![0_usize; centers.len()]; + let mut numerics = vec![0_usize; centers.len()]; + + for (cluster_index, xs) in clusters.iter().enumerate() { + for x in xs { + let Some(flags) = numeric_flags_by_x.get_mut(&x.to_bits()) else { + continue; + }; + let Some(is_numeric) = flags.pop_front() else { + continue; + }; + totals[cluster_index] += 1; + if is_numeric { + numerics[cluster_index] += 1; } } } - ClusterInfo { - center, - count: total, - numeric_frac: if total > 0 { - numeric as f32 / total as f32 - } else { - 0.0 - }, - } - }; - // Merge distance: allow merging clusters that are slightly beyond the - // original threshold. Use 1.5× threshold to catch header-vs-data splits. - let merge_dist = threshold * 1.5; + centers + .into_iter() + .zip(totals) + .zip(numerics) + .map(|((center, count), numeric)| ClusterInfo { + center, + count, + numeric_frac: if count > 0 { + numeric as f32 / count as f32 + } else { + 0.0 + }, + }) + .collect() + }; // Iterate and merge adjacent pairs. Use a simple left-to-right scan. let mut merged = true; while merged { merged = false; let mut i = 0; + let mut flags_for_pass = numeric_flags_by_x.clone(); + let infos = compute_infos(&mut flags_for_pass, &clusters); while i + 1 < clusters.len() { - let info_a = compute_info(&clusters[i]); - let info_b = compute_info(&clusters[i + 1]); + let info_a = &infos[i]; + let info_b = &infos[i + 1]; let dist = (info_b.center - info_a.center).abs(); - if dist > merge_dist { + if dist > threshold * 1.5 { i += 1; continue; } @@ -238,30 +265,23 @@ fn merge_numeric_adjacent_clusters( (&info_b, &info_a) }; - // Merge if the dense cluster is predominantly numeric (>50%) - // and the sparse cluster has at most 1/3 the items of the dense one. - let should_merge = - dense.numeric_frac > 0.50 && sparse.count <= dense.count / 2 && sparse.count <= 5; + // Merge if the dense cluster is predominantly numeric (>50%), the + // sparse cluster is header-like (non-numeric), and the sparse cluster + // has at most 1/3 the items of the dense one. Two numeric clusters + // are separate value columns even when one has fewer populated rows. + let should_merge = dense.numeric_frac > 0.50 + && sparse.numeric_frac <= 0.50 + && sparse.count <= dense.count / 2 + && sparse.count <= 5; if should_merge { - log::debug!( - " merging column clusters: center {:.1} ({} items, {:.0}% numeric) + {:.1} ({} items, {:.0}% numeric), dist={:.1}", - info_a.center, - info_a.count, - info_a.numeric_frac * 100.0, - info_b.center, - info_b.count, - info_b.numeric_frac * 100.0, - dist, - ); - // Merge cluster i+1 into cluster i - let next = clusters.remove(i + 1); - clusters[i].extend(next); + let mut merged_cluster = clusters.remove(i + 1); + clusters[i].append(&mut merged_cluster); merged = true; - // Don't increment i — check if the merged cluster can merge further - } else { - i += 1; + break; } + + i += 1; } } @@ -656,6 +676,85 @@ mod tests { assert!(cols.len() <= 1); } + #[test] + fn test_merge_numeric_adjacent_clusters_keeps_numeric_clusters_separate() { + let mut items_data = vec![ + make_item("100", 100.0, 500.0, 10.0), + make_item("200", 100.0, 480.0, 10.0), + ]; + for row in 0..10 { + items_data.push(make_item( + &format!("3{:02}", row), + 130.0, + 460.0 - row as f32 * 20.0, + 10.0, + )); + } + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let clusters = vec![vec![100.0, 100.0], vec![130.0; 10]]; + + let merged = merge_numeric_adjacent_clusters(clusters, &items, 25.0); + + assert_eq!( + merged.len(), + 2, + "two predominantly numeric columns must remain separate" + ); + } + + #[test] + fn test_merge_numeric_adjacent_clusters_uses_cluster_membership() { + let mut items_data = vec![make_item("100", 100.0, 500.0, 10.0)]; + items_data.push(make_item("Header", 110.0, 480.0, 10.0)); + items_data.push(make_item("Label", 112.0, 460.0, 10.0)); + for row in 0..10 { + items_data.push(make_item( + &format!("3{:02}", row), + 130.0, + 440.0 - row as f32 * 20.0, + 10.0, + )); + } + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let mut dense_cluster = vec![110.0, 112.0]; + dense_cluster.extend(std::iter::repeat(130.0).take(10)); + let clusters = vec![vec![100.0], dense_cluster]; + + let merged = merge_numeric_adjacent_clusters(clusters, &items, 25.0); + + assert_eq!( + merged.len(), + 2, + "jitter from the dense cluster must not reclassify the sparse numeric cluster" + ); + } + + #[test] + fn test_merge_numeric_adjacent_clusters_merges_header_with_numeric_data() { + let mut items_data = vec![ + make_item("Value", 100.0, 500.0, 10.0), + make_item("(USD)", 100.0, 480.0, 10.0), + ]; + for row in 0..10 { + items_data.push(make_item( + &format!("1{:02}", row), + 130.0, + 460.0 - row as f32 * 20.0, + 10.0, + )); + } + let items: Vec<(usize, &TextItem)> = items_data.iter().enumerate().collect(); + let clusters = vec![vec![100.0, 100.0], vec![130.0; 10]]; + + let merged = merge_numeric_adjacent_clusters(clusters, &items, 25.0); + + assert_eq!( + merged.len(), + 1, + "a sparse text header cluster should merge with its numeric data cluster" + ); + } + // --- find_row_boundaries --- #[test]