From 6dec68c66b47ef38c9612786094a51affedacb03 Mon Sep 17 00:00:00 2001 From: riz-adnan Date: Thu, 16 Jul 2026 21:18:28 +0530 Subject: [PATCH] feat: expose OCR failure details in parse results --- README.md | 11 +++++ crates/liteparse-napi/src/types.rs | 22 +++++++++ crates/liteparse-python/src/cli.rs | 23 ++++++++-- crates/liteparse-python/src/lib.rs | 36 ++++++++++++++- crates/liteparse-wasm/src/lib.rs | 19 ++++++++ crates/liteparse/src/main.rs | 25 +++++++++-- crates/liteparse/src/ocr_merge.rs | 64 +++++++++++++++++++++++---- crates/liteparse/src/output/json.rs | 46 +++++++++++++++---- crates/liteparse/src/parser.rs | 13 +++++- packages/node/native.d.ts | 6 +++ packages/node/src/cli.ts | 14 ++++++ packages/node/src/lib.ts | 13 ++++++ packages/node/src/native.ts | 7 +++ packages/python/liteparse/__init__.py | 2 + packages/python/liteparse/parser.py | 6 +++ packages/python/liteparse/types.py | 11 +++++ 16 files changed, 292 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index afc18c47..c04fa071 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,9 @@ lit parse document.pdf --target-pages "1-5,10,15-20" # Parse without OCR lit parse document.pdf --no-ocr +# Continue if OCR fails and report failed pages in JSON +lit parse document.pdf --format json --ocr-failure-non-fatal + # Parse a remote PDF curl -sL https://example.com/report.pdf | lit parse - ``` @@ -265,6 +268,7 @@ Options: --preserve-small-text Keep very small text --password Password for encrypted documents --num-workers Concurrent OCR workers [default: CPU cores - 1] + --ocr-failure-non-fatal Continue if OCR fails; JSON includes failed OCR pages -q, --quiet Suppress progress output -h, --help Print help ``` @@ -286,6 +290,7 @@ Options: --extension Only process files with this extension (e.g., ".pdf") --password Password for encrypted documents --num-workers Concurrent OCR workers + --ocr-failure-non-fatal Continue if OCR fails; JSON includes failed OCR pages -q, --quiet Suppress progress output -h, --help Print help ``` @@ -346,6 +351,12 @@ Or pass the path directly: lit parse document.pdf --tessdata-path /path/to/tessdata ``` +By default, a systemic OCR failure on text-sparse pages aborts parsing so OCR setup +problems are visible. If you prefer partial output, pass +`--ocr-failure-non-fatal`; JSON output includes `failed_ocr_pages`, a list of +1-based page numbers whose OCR pass failed, and `ocr_failures`, objects with the +page number and OCR engine error string. + ### Optional: HTTP OCR Servers For higher accuracy or better performance, you can use an HTTP OCR server. We provide ready-to-use example wrappers for popular OCR engines: diff --git a/crates/liteparse-napi/src/types.rs b/crates/liteparse-napi/src/types.rs index bbe9ad2d..28bb8847 100644 --- a/crates/liteparse-napi/src/types.rs +++ b/crates/liteparse-napi/src/types.rs @@ -440,9 +440,18 @@ impl JsParsedPage { pub struct JsParseResult { pub pages: Vec, pub text: String, + pub failed_ocr_pages: Vec, + pub ocr_failures: Vec, pub images: Vec, } +#[napi(object)] +#[derive(Clone)] +pub struct JsOcrFailure { + pub page_number: u32, + pub error: String, +} + #[napi(object)] #[derive(Clone)] pub struct JsExtractedImage { @@ -512,6 +521,19 @@ impl JsParseResult { Self { pages: result.pages.iter().map(JsParsedPage::from_rust).collect(), text: result.text.clone(), + failed_ocr_pages: result + .failed_ocr_pages + .iter() + .map(|&page| page as u32) + .collect(), + ocr_failures: result + .ocr_failures + .iter() + .map(|failure| JsOcrFailure { + page_number: failure.page_number as u32, + error: failure.error.clone(), + }) + .collect(), images: result .images .iter() diff --git a/crates/liteparse-python/src/cli.rs b/crates/liteparse-python/src/cli.rs index b13ab5ca..a4633730 100644 --- a/crates/liteparse-python/src/cli.rs +++ b/crates/liteparse-python/src/cli.rs @@ -58,6 +58,9 @@ struct ParseCommand { quiet: bool, #[arg(long)] num_workers: Option, + /// Continue parsing when OCR fails and report failed pages in JSON output. + #[arg(long)] + ocr_failure_non_fatal: bool, /// How to surface raster images in markdown output: `off`, `placeholder` /// (default), or `embed` (extracts PNG bytes, written next to the output /// when `--image-output-dir` is set). @@ -125,6 +128,9 @@ struct BatchParseCommand { quiet: bool, #[arg(long)] num_workers: Option, + /// Continue parsing when OCR fails and report failed pages in JSON output. + #[arg(long)] + ocr_failure_non_fatal: bool, /// Include per-page complexity signals as a `complexity` object on each /// page of JSON output. Off by default. #[arg(long)] @@ -189,6 +195,7 @@ pub fn run_cli(args: Vec) -> Result<(), Box> { quiet: cmd.quiet, ocr_server_url: cmd.ocr_server_url, ocr_server_headers: cmd.ocr_server_headers, + ocr_failure_fatal: !cmd.ocr_failure_non_fatal, image_mode, extract_links: !cmd.no_links, include_complexity: cmd.complexity, @@ -200,7 +207,11 @@ pub fn run_cli(args: Vec) -> Result<(), Box> { let lp = LiteParse::new(config); let result = rt.block_on(lp.parse(&cmd.file))?; let formatted = match lp.config().output_format { - OutputFormat::Json => json::format_json(&result.pages)?, + OutputFormat::Json => json::format_json( + &result.pages, + &result.failed_ocr_pages, + &result.ocr_failures, + )?, OutputFormat::Text => text::format_text(&result.pages), OutputFormat::Markdown => result.text.clone(), }; @@ -287,6 +298,7 @@ pub fn run_cli(args: Vec) -> Result<(), Box> { quiet: cmd.quiet, ocr_server_url: cmd.ocr_server_url, ocr_server_headers: cmd.ocr_server_headers, + ocr_failure_fatal: !cmd.ocr_failure_non_fatal, include_complexity: cmd.complexity, ..Default::default() }; @@ -328,9 +340,12 @@ pub fn run_cli(args: Vec) -> Result<(), Box> { Ok(result) => { let fmt_result: Result> = match lp.config().output_format { - OutputFormat::Json => { - json::format_json(&result.pages).map_err(|e| e.into()) - } + OutputFormat::Json => json::format_json( + &result.pages, + &result.failed_ocr_pages, + &result.ocr_failures, + ) + .map_err(|e| e.into()), OutputFormat::Text => Ok(text::format_text(&result.pages)), OutputFormat::Markdown => Ok(result.text.clone()), }; diff --git a/crates/liteparse-python/src/lib.rs b/crates/liteparse-python/src/lib.rs index a089d570..605361a3 100644 --- a/crates/liteparse-python/src/lib.rs +++ b/crates/liteparse-python/src/lib.rs @@ -180,9 +180,32 @@ struct PyParseResult { #[pyo3(get)] text: String, #[pyo3(get)] + failed_ocr_pages: Vec, + #[pyo3(get)] + ocr_failures: Vec, + #[pyo3(get)] images: Vec, } +#[pyclass(frozen, from_py_object)] +#[derive(Clone)] +struct PyOcrFailure { + #[pyo3(get)] + page_number: usize, + #[pyo3(get)] + error: String, +} + +#[pymethods] +impl PyOcrFailure { + fn __repr__(&self) -> String { + format!( + "OcrFailure(page_number={}, error={:?})", + self.page_number, self.error + ) + } +} + #[pymethods] impl PyParseResult { #[getter] @@ -196,9 +219,10 @@ impl PyParseResult { fn __repr__(&self) -> String { format!( - "ParseResult(pages={}, text_len={}, images={})", + "ParseResult(pages={}, text_len={}, failed_ocr_pages={}, images={})", self.pages.len(), self.text.len(), + self.failed_ocr_pages.len(), self.images.len() ) } @@ -213,6 +237,15 @@ impl PyParseResult { .map(PyParsedPage::from_rust) .collect(), text: result.text, + failed_ocr_pages: result.failed_ocr_pages, + ocr_failures: result + .ocr_failures + .into_iter() + .map(|failure| PyOcrFailure { + page_number: failure.page_number, + error: failure.error, + }) + .collect(), images: result .images .into_iter() @@ -693,6 +726,7 @@ fn _liteparse(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/crates/liteparse-wasm/src/lib.rs b/crates/liteparse-wasm/src/lib.rs index a2db51b9..34ab894f 100644 --- a/crates/liteparse-wasm/src/lib.rs +++ b/crates/liteparse-wasm/src/lib.rs @@ -273,9 +273,19 @@ pub struct ParsedPage { pub struct ParseResult { pub pages: Vec, pub text: String, + pub failed_ocr_pages: Vec, + pub ocr_failures: Vec, pub images: Vec, } +#[derive(Serialize, Tsify)] +#[tsify(into_wasm_abi)] +#[serde(rename_all = "camelCase")] +pub struct OcrFailure { + pub page_number: usize, + pub error: String, +} + #[derive(Serialize, Tsify)] #[tsify(into_wasm_abi)] #[serde(rename_all = "camelCase")] @@ -520,6 +530,15 @@ impl LiteParse { Ok(ParseResult { pages, text: result.text.clone(), + failed_ocr_pages: result.failed_ocr_pages.clone(), + ocr_failures: result + .ocr_failures + .iter() + .map(|failure| OcrFailure { + page_number: failure.page_number, + error: failure.error.clone(), + }) + .collect(), images, }) } diff --git a/crates/liteparse/src/main.rs b/crates/liteparse/src/main.rs index 241616ac..d8679d09 100644 --- a/crates/liteparse/src/main.rs +++ b/crates/liteparse/src/main.rs @@ -98,6 +98,10 @@ struct ParseCommand { #[arg(long)] num_workers: Option, + /// Continue parsing when OCR fails and report failed pages in JSON output + #[arg(long)] + ocr_failure_non_fatal: bool, + /// How to surface raster images in markdown output: /// `off` strips them, `placeholder` (default) emits `![](image_pN_K.png)` /// references in reading order, `embed` extracts each image's PNG bytes @@ -214,6 +218,10 @@ struct BatchParseCommand { #[arg(long)] num_workers: Option, + /// Continue parsing when OCR fails and report failed pages in JSON output + #[arg(long)] + ocr_failure_non_fatal: bool, + /// Include per-page complexity signals as a `complexity` object on each /// page of JSON output. Off by default. #[arg(long)] @@ -317,6 +325,7 @@ async fn main() -> Result<(), Box> { quiet: cmd.quiet, ocr_server_url: cmd.ocr_server_url, ocr_server_headers: cmd.ocr_server_headers, + ocr_failure_fatal: !cmd.ocr_failure_non_fatal, image_mode, extract_links: !cmd.no_links, include_complexity: cmd.complexity, @@ -329,7 +338,11 @@ async fn main() -> Result<(), Box> { let lp = LiteParse::new(config); let result = lp.parse(&cmd.file).await?; let formatted = match lp.config().output_format { - OutputFormat::Json => json::format_json(&result.pages)?, + OutputFormat::Json => json::format_json( + &result.pages, + &result.failed_ocr_pages, + &result.ocr_failures, + )?, OutputFormat::Text => text::format_text(&result.pages), OutputFormat::Markdown => result.text.clone(), }; @@ -420,6 +433,7 @@ async fn main() -> Result<(), Box> { quiet: cmd.quiet, ocr_server_url: cmd.ocr_server_url, ocr_server_headers: cmd.ocr_server_headers, + ocr_failure_fatal: !cmd.ocr_failure_non_fatal, include_complexity: cmd.complexity, ..Default::default() }; @@ -464,9 +478,12 @@ async fn main() -> Result<(), Box> { Ok(result) => { let fmt_result: Result> = match lp.config().output_format { - OutputFormat::Json => { - json::format_json(&result.pages).map_err(|e| e.into()) - } + OutputFormat::Json => json::format_json( + &result.pages, + &result.failed_ocr_pages, + &result.ocr_failures, + ) + .map_err(|e| e.into()), OutputFormat::Text => Ok(text::format_text(&result.pages)), OutputFormat::Markdown => Ok(result.text.clone()), }; diff --git a/crates/liteparse/src/ocr_merge.rs b/crates/liteparse/src/ocr_merge.rs index 5b963c41..f3292dc3 100644 --- a/crates/liteparse/src/ocr_merge.rs +++ b/crates/liteparse/src/ocr_merge.rs @@ -113,6 +113,12 @@ pub struct PageComplexityStats { pub reasons: Vec, } +#[derive(Debug, Clone, Serialize)] +pub struct OcrFailure { + pub page_number: usize, + pub error: String, +} + pub(crate) fn calculate_page_complexity( page: &Page, page_obj: &pdfium::Page, @@ -283,7 +289,7 @@ pub(crate) async fn ocr_and_merge_rendered( ocr_language: &str, num_workers: usize, ocr_failure_fatal: bool, -) -> Result<(), LiteParseError> { +) -> Result, LiteParseError> { // Phase 1: spawn one async task per page. A semaphore limits how many run // `recognize` concurrently to `num_workers`. // @@ -353,6 +359,7 @@ pub(crate) async fn ocr_and_merge_rendered( // a broken OCR setup would abort perfectly good native-text documents. let total_tasks = handles.len(); let mut failed_tasks = 0usize; + let mut ocr_failures = Vec::new(); let mut failed_sparse_text_page = false; let mut first_error: Option = None; @@ -361,11 +368,15 @@ pub(crate) async fn ocr_and_merge_rendered( Ok(Ok(results)) => results, Ok(Err(e)) => { failed_tasks += 1; + let msg = e.to_string(); + ocr_failures.push(OcrFailure { + page_number, + error: msg.clone(), + }); failed_sparse_text_page |= page_has_sparse_native_text(&pages[idx]); // Only log the first failure to avoid flooding stderr with an // identical message for every page. if first_error.is_none() { - let msg = e.to_string(); eprintln!("[ocr] failed for page {}: {}", page_number, msg); first_error = Some(msg); } @@ -373,9 +384,13 @@ pub(crate) async fn ocr_and_merge_rendered( } Err(e) => { failed_tasks += 1; + let msg = e.to_string(); + ocr_failures.push(OcrFailure { + page_number, + error: msg.clone(), + }); failed_sparse_text_page |= page_has_sparse_native_text(&pages[idx]); if first_error.is_none() { - let msg = e.to_string(); eprintln!("[ocr] task panicked for page {}: {}", page_number, msg); first_error = Some(msg); } @@ -496,8 +511,13 @@ pub(crate) async fn ocr_and_merge_rendered( let detail = first_error.unwrap_or_else(|| "unknown error".to_string()); if ocr_failure_fatal { return Err(LiteParseError::Ocr(format!( - "OCR failed for all {} page(s): {}", - total_tasks, detail + "OCR failed for all {} page(s) {:?}: {}", + total_tasks, + ocr_failures + .iter() + .map(|f| f.page_number) + .collect::>(), + detail ))); } // Non-fatal mode: the caller prefers partial results over a hard abort, @@ -512,12 +532,17 @@ pub(crate) async fn ocr_and_merge_rendered( // Surface a concise summary for partial failures without flooding stderr. if failed_tasks > 0 { eprintln!( - "[ocr] {}/{} page(s) failed OCR; continuing with partial results", - failed_tasks, total_tasks + "[ocr] {}/{} page(s) failed OCR {:?}; continuing with partial results", + failed_tasks, + total_tasks, + ocr_failures + .iter() + .map(|f| f.page_number) + .collect::>() ); } - Ok(()) + Ok(ocr_failures) } /// True when the page's native (already-extracted) text is sparse enough that @@ -978,6 +1003,10 @@ mod tests { } } + fn failure_pages(failures: &[OcrFailure]) -> Vec { + failures.iter().map(|f| f.page_number).collect() + } + // When every OCR task fails (e.g. missing language data), the function must // return an error instead of silently reporting success with no OCR text. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -995,6 +1024,10 @@ mod tests { msg.contains("OCR failed for all 2 page(s)"), "unexpected error message: {msg}" ); + assert!( + msg.contains("[1, 2]"), + "error should include failed page numbers: {msg}" + ); assert!( msg.contains("traineddata"), "error should carry the underlying cause: {msg}" @@ -1012,6 +1045,7 @@ mod tests { ocr_and_merge_rendered(&mut pages, Vec::new(), 72.0, engine, "eng", 2, true).await; assert!(result.is_ok(), "empty OCR set should succeed: {result:?}"); + assert!(result.unwrap().is_empty()); } // Regression guard: when OCR fails but every failing page already had native @@ -1030,6 +1064,9 @@ mod tests { result.is_ok(), "OCR failure on already-native-text pages must not abort the parse: {result:?}" ); + let failures = result.unwrap(); + assert_eq!(failure_pages(&failures), vec![1, 2]); + assert!(failures[0].error.contains("traineddata")); // Native text is preserved untouched. assert_eq!(pages[0].text_items.len(), 1); assert_eq!(pages[1].text_items.len(), 1); @@ -1052,6 +1089,10 @@ mod tests { err.to_string().contains("OCR failed for all 2 page(s)"), "unexpected error message: {err}" ); + assert!( + err.to_string().contains("[1, 2]"), + "error should include failed page numbers: {err}" + ); } // Regression guard for the review finding: low-coverage pages are rendered @@ -1071,6 +1112,10 @@ mod tests { err.to_string().contains("OCR failed for all 1 page(s)"), "unexpected error message: {err}" ); + assert!( + err.to_string().contains("[1]"), + "error should include failed page numbers: {err}" + ); } // With `ocr_failure_fatal = false`, a systemic OCR failure that would @@ -1090,6 +1135,9 @@ mod tests { result.is_ok(), "non-fatal mode must not abort on systemic OCR failure: {result:?}" ); + let failures = result.unwrap(); + assert_eq!(failure_pages(&failures), vec![1, 2]); + assert!(failures[1].error.contains("traineddata")); // The native-text page keeps its text; the blank page simply has no OCR. assert_eq!(pages[0].text_items.len(), 1); } diff --git a/crates/liteparse/src/output/json.rs b/crates/liteparse/src/output/json.rs index 980de9d5..404e4532 100644 --- a/crates/liteparse/src/output/json.rs +++ b/crates/liteparse/src/output/json.rs @@ -1,4 +1,4 @@ -use crate::ocr_merge::PageComplexityStats; +use crate::ocr_merge::{OcrFailure, PageComplexityStats}; use crate::types::ParsedPage; use serde::Serialize; @@ -30,12 +30,20 @@ pub(crate) struct JsonPage { #[derive(Debug, Serialize)] pub(crate) struct ParseResultJson { + pub failed_ocr_pages: Vec, + pub ocr_failures: Vec, pub pages: Vec, } /// Build structured JSON output from parsed pages. -pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson { +pub(crate) fn build_json( + pages: &[ParsedPage], + failed_ocr_pages: &[usize], + ocr_failures: &[OcrFailure], +) -> ParseResultJson { ParseResultJson { + failed_ocr_pages: failed_ocr_pages.to_vec(), + ocr_failures: ocr_failures.to_vec(), pages: pages .iter() .map(|page| JsonPage { @@ -64,8 +72,12 @@ pub(crate) fn build_json(pages: &[ParsedPage]) -> ParseResultJson { } /// Format parsed pages as pretty-printed JSON string. -pub fn format_json(pages: &[ParsedPage]) -> Result { - let result = build_json(pages); +pub fn format_json( + pages: &[ParsedPage], + failed_ocr_pages: &[usize], + ocr_failures: &[OcrFailure], +) -> Result { + let result = build_json(pages, failed_ocr_pages, ocr_failures); serde_json::to_string_pretty(&result) } @@ -108,7 +120,9 @@ mod tests { #[test] fn test_build_json_native_text_defaults_confidence_to_one() { - let j = build_json(&[page(vec![item("hi", None)])]); + let j = build_json(&[page(vec![item("hi", None)])], &[], &[]); + assert!(j.failed_ocr_pages.is_empty()); + assert!(j.ocr_failures.is_empty()); assert_eq!(j.pages.len(), 1); assert_eq!(j.pages[0].page, 1); assert_eq!(j.pages[0].text_items[0].confidence, Some(1.0)); @@ -117,21 +131,37 @@ mod tests { #[test] fn test_build_json_preserves_ocr_confidence() { - let j = build_json(&[page(vec![item("hi", Some(0.42))])]); + let failures = vec![OcrFailure { + page_number: 2, + error: "missing traineddata".into(), + }]; + let j = build_json(&[page(vec![item("hi", Some(0.42))])], &[2], &failures); + assert_eq!(j.failed_ocr_pages, vec![2]); + assert_eq!(j.ocr_failures[0].error, "missing traineddata"); assert_eq!(j.pages[0].text_items[0].confidence, Some(0.42)); } #[test] fn test_format_json_pretty() { - let s = format_json(&[page(vec![item("hi", None)])]).unwrap(); + let failures = vec![OcrFailure { + page_number: 3, + error: "OCR timeout".into(), + }]; + let s = format_json(&[page(vec![item("hi", None)])], &[3], &failures).unwrap(); assert!(s.contains("\n")); + assert!(s.contains("\"failed_ocr_pages\"")); + assert!(s.contains("\"ocr_failures\"")); + assert!(s.contains("OCR timeout")); + assert!(s.contains("3")); assert!(s.contains("\"text\": \"hi\"")); assert!(s.contains("\"page\": 1")); } #[test] fn test_build_json_empty() { - let j = build_json(&[]); + let j = build_json(&[], &[], &[]); + assert!(j.failed_ocr_pages.is_empty()); + assert!(j.ocr_failures.is_empty()); assert!(j.pages.is_empty()); } } diff --git a/crates/liteparse/src/parser.rs b/crates/liteparse/src/parser.rs index dd5aa17f..a4fa18ce 100644 --- a/crates/liteparse/src/parser.rs +++ b/crates/liteparse/src/parser.rs @@ -9,6 +9,7 @@ use crate::ocr::http_simple::HttpOcrEngine; #[cfg(feature = "tesseract")] use crate::ocr::tesseract::TesseractOcrEngine; use crate::ocr_merge; +use crate::ocr_merge::OcrFailure; use crate::output::markdown; use crate::projection; #[cfg(not(target_arch = "wasm32"))] @@ -22,6 +23,10 @@ pub struct ParseResult { pub pages: Vec, /// Full document text, concatenated from all pages. pub text: String, + /// 1-based page numbers whose OCR task failed while parsing continued. + pub failed_ocr_pages: Vec, + /// OCR failures with page numbers and engine error messages. + pub ocr_failures: Vec, /// Document outline (bookmarks) when present. Used by the markdown /// emitter as a high-priority heading source on untagged PDFs. pub outline: Vec, @@ -343,8 +348,9 @@ impl LiteParse { let t1 = web_time::Instant::now(); // OCR pass (engine resolved before the render block above). + let mut ocr_failures = Vec::new(); if let Some(engine) = ocr_engine { - ocr_merge::ocr_and_merge_rendered( + ocr_failures = ocr_merge::ocr_and_merge_rendered( &mut pages, ocr_rendered, self.config.dpi, @@ -355,6 +361,7 @@ impl LiteParse { ) .await?; } + let failed_ocr_pages = ocr_failures.iter().map(|f| f.page_number).collect(); let t_ocr = web_time::Instant::now(); log(&format!( "[liteparse] ocr: {:.1}ms", @@ -410,6 +417,8 @@ impl LiteParse { Ok(ParseResult { pages: parsed_pages, text: full_text, + failed_ocr_pages, + ocr_failures, outline, images, }) @@ -445,6 +454,8 @@ impl LiteParse { ParseResult { pages: parsed_pages, text: full_text, + failed_ocr_pages: Vec::new(), + ocr_failures: Vec::new(), outline, images: Vec::new(), } diff --git a/packages/node/native.d.ts b/packages/node/native.d.ts index 7ba78b76..4b603e35 100644 --- a/packages/node/native.d.ts +++ b/packages/node/native.d.ts @@ -163,8 +163,14 @@ export interface JsParsedPage { export interface JsParseResult { pages: Array text: string + failedOcrPages: Array + ocrFailures: Array images: Array } +export interface JsOcrFailure { + pageNumber: number + error: string +} export interface JsExtractedImage { id: string page: number diff --git a/packages/node/src/cli.ts b/packages/node/src/cli.ts index 79ccce9d..27883286 100644 --- a/packages/node/src/cli.ts +++ b/packages/node/src/cli.ts @@ -61,6 +61,10 @@ program .option("--config ", "JSON config file path") .option("-q, --quiet", "Suppress progress output") .option("--num-workers ", "Number of concurrent OCR workers", parseInt) + .option( + "--ocr-failure-non-fatal", + "Continue parsing when OCR fails and report failed pages in JSON output", + ) .option( "--complexity", "Include per-page complexity signals in JSON output", @@ -95,6 +99,7 @@ program if (opts.password) config.password = opts.password as string; if (opts.quiet) config.quiet = true; if (opts.numWorkers) config.numWorkers = opts.numWorkers as number; + if (opts.ocrFailureNonFatal) config.ocrFailureFatal = false; if (opts.complexity) config.includeComplexity = true; // Default CLI output to text (library defaults to json) @@ -107,6 +112,8 @@ program config.outputFormat === "json" ? JSON.stringify( { + failedOcrPages: result.failedOcrPages, + ocrFailures: result.ocrFailures, pages: result.pages.map((p) => ({ page: p.pageNum, width: p.width, @@ -287,6 +294,10 @@ program .option("--password ", "Password for encrypted documents") .option("-q, --quiet", "Suppress progress output") .option("--num-workers ", "Number of concurrent OCR workers", parseInt) + .option( + "--ocr-failure-non-fatal", + "Continue parsing when OCR fails and report failed pages in JSON output", + ) .action( async ( inputDir: string, @@ -308,6 +319,7 @@ program if (opts.password) config.password = opts.password as string; if (opts.quiet) config.quiet = true; if (opts.numWorkers) config.numWorkers = opts.numWorkers as number; + if (opts.ocrFailureNonFatal) config.ocrFailureFatal = false; const parser = new LiteParse(config); const outExt = format === "json" ? ".json" : format === "markdown" ? ".md" : ".txt"; @@ -358,6 +370,8 @@ program format === "json" ? JSON.stringify( { + failedOcrPages: result.failedOcrPages, + ocrFailures: result.ocrFailures, pages: result.pages.map((p) => ({ page: p.pageNum, width: p.width, diff --git a/packages/node/src/lib.ts b/packages/node/src/lib.ts index c28a9bab..22c0bc0d 100644 --- a/packages/node/src/lib.ts +++ b/packages/node/src/lib.ts @@ -187,10 +187,19 @@ export interface ExtractedImage { export interface ParseResult { pages: ParsedPage[]; text: string; + /** 1-based page numbers whose OCR task failed while parsing continued. */ + failedOcrPages: number[]; + /** OCR failures with page numbers and engine error messages. */ + ocrFailures: OcrFailure[]; /** Populated only when configured with `imageMode: "embed"`. */ images: ExtractedImage[]; } +export interface OcrFailure { + pageNumber: number; + error: string; +} + export interface ScreenshotResult { pageNum: number; width: number; @@ -306,6 +315,8 @@ export class LiteParse { return { pages: result.pages.map(toPage), text: result.text, + failedOcrPages: result.failedOcrPages ?? [], + ocrFailures: result.ocrFailures ?? [], images: (result.images ?? []).map(toImage), }; } @@ -328,6 +339,8 @@ export class LiteParse { return { pages: result.pages.map(toPage), text: result.text, + failedOcrPages: result.failedOcrPages ?? [], + ocrFailures: result.ocrFailures ?? [], images: (result.images ?? []).map(toImage), }; } diff --git a/packages/node/src/native.ts b/packages/node/src/native.ts index 22d8accb..32334603 100644 --- a/packages/node/src/native.ts +++ b/packages/node/src/native.ts @@ -116,9 +116,16 @@ export interface NativeExtractedImage { export interface NativeParseResult { pages: NativeParsedPage[]; text: string; + failedOcrPages: number[]; + ocrFailures: NativeOcrFailure[]; images: NativeExtractedImage[]; } +export interface NativeOcrFailure { + pageNumber: number; + error: string; +} + export interface NativeScreenshotResult { pageNum: number; width: number; diff --git a/packages/python/liteparse/__init__.py b/packages/python/liteparse/__init__.py index 53e99d77..8be9759b 100644 --- a/packages/python/liteparse/__init__.py +++ b/packages/python/liteparse/__init__.py @@ -4,6 +4,7 @@ from .types import ( ExtractedImage, LiteParseConfig, + OcrFailure, PageComplexityStats, ParseResult, ParsedPage, @@ -26,6 +27,7 @@ "WordBox", "ScreenshotResult", "PageComplexityStats", + "OcrFailure", "ExtractedImage", "ParseError", "search_items", diff --git a/packages/python/liteparse/parser.py b/packages/python/liteparse/parser.py index f3efe97b..1e9ee1b8 100644 --- a/packages/python/liteparse/parser.py +++ b/packages/python/liteparse/parser.py @@ -9,6 +9,7 @@ from .types import ( ExtractedImage, LiteParseConfig, + OcrFailure, PageComplexityStats, ParsedPage, ParseError, @@ -94,6 +95,11 @@ def _convert_native_result(native_result: Any) -> ParseResult: return ParseResult( pages=pages, text=native_result.text, + failed_ocr_pages=list(getattr(native_result, "failed_ocr_pages", [])), + ocr_failures=[ + OcrFailure(page_number=f.page_number, error=f.error) + for f in getattr(native_result, "ocr_failures", []) + ], images=images, ) diff --git a/packages/python/liteparse/types.py b/packages/python/liteparse/types.py index a0809640..705b97b4 100644 --- a/packages/python/liteparse/types.py +++ b/packages/python/liteparse/types.py @@ -69,6 +69,10 @@ class ParseResult: pages: List[ParsedPage] text: str images: List[ExtractedImage] = field(default_factory=list) + #: 1-based page numbers whose OCR task failed while parsing continued. + failed_ocr_pages: List[int] = field(default_factory=list) + #: OCR failures with page numbers and engine error messages. + ocr_failures: List[OcrFailure] = field(default_factory=list) @property def num_pages(self) -> int: @@ -109,6 +113,13 @@ class PageComplexityStats: reasons: list[str] +@dataclass +class OcrFailure: + """A failed OCR page and the error reported by the OCR engine.""" + page_number: int + error: str + + @dataclass class LiteParseConfig: """Resolved parser configuration."""