Skip to content

Expose failed OCR page numbers in parse results #358

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
riz-adnan wants to merge 1 commit into from
Closed

Expose failed OCR page numbers in parse results #358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ lit parse document.pdf --target-pages "1-5,10,15-20"
# Parse without OCR
lit parse document.pdf --no-ocr

# Continue if OCR fails and report failed pages in JSON
lit parse document.pdf --format json --ocr-failure-non-fatal

# Parse a remote PDF
curl -sL https://example.com/report.pdf | lit parse -
```
Expand Down Expand Up @@ -265,6 +268,7 @@ Options:
--preserve-small-text Keep very small text
--password <password> Password for encrypted documents
--num-workers <n> Concurrent OCR workers [default: CPU cores - 1]
--ocr-failure-non-fatal Continue if OCR fails; JSON includes failed OCR pages
-q, --quiet Suppress progress output
-h, --help Print help
```
Expand All @@ -286,6 +290,7 @@ Options:
--extension <ext> Only process files with this extension (e.g., ".pdf")
--password <password> Password for encrypted documents
--num-workers <n> Concurrent OCR workers
--ocr-failure-non-fatal Continue if OCR fails; JSON includes failed OCR pages
-q, --quiet Suppress progress output
-h, --help Print help
```
Expand Down Expand Up @@ -346,6 +351,12 @@ Or pass the path directly:
lit parse document.pdf --tessdata-path /path/to/tessdata
```

By default, a systemic OCR failure on text-sparse pages aborts parsing so OCR setup
problems are visible. If you prefer partial output, pass
`--ocr-failure-non-fatal`; JSON output includes `failed_ocr_pages`, a list of
1-based page numbers whose OCR pass failed, and `ocr_failures`, objects with the
page number and OCR engine error string.

### Optional: HTTP OCR Servers

For higher accuracy or better performance, you can use an HTTP OCR server. We provide ready-to-use example wrappers for popular OCR engines:
Expand Down
22 changes: 22 additions & 0 deletions crates/liteparse-napi/src/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -440,9 +440,18 @@ impl JsParsedPage {
pub struct JsParseResult {
pub pages: Vec<JsParsedPage>,
pub text: String,
pub failed_ocr_pages: Vec<u32>,
pub ocr_failures: Vec<JsOcrFailure>,
pub images: Vec<JsExtractedImage>,
}

#[napi(object)]
#[derive(Clone)]
pub struct JsOcrFailure {
pub page_number: u32,
pub error: String,
}

#[napi(object)]
#[derive(Clone)]
pub struct JsExtractedImage {
Expand Down Expand Up @@ -512,6 +521,19 @@ impl JsParseResult {
Self {
pages: result.pages.iter().map(JsParsedPage::from_rust).collect(),
text: result.text.clone(),
failed_ocr_pages: result
.failed_ocr_pages
.iter()
.map(|&page| page as u32)
.collect(),
ocr_failures: result
.ocr_failures
.iter()
.map(|failure| JsOcrFailure {
page_number: failure.page_number as u32,
error: failure.error.clone(),
})
.collect(),
images: result
.images
.iter()
Expand Down
23 changes: 19 additions & 4 deletions crates/liteparse-python/src/cli.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ struct ParseCommand {
quiet: bool,
#[arg(long)]
num_workers: Option<usize>,
/// Continue parsing when OCR fails and report failed pages in JSON output.
#[arg(long)]
ocr_failure_non_fatal: bool,
/// How to surface raster images in markdown output: `off`, `placeholder`
/// (default), or `embed` (extracts PNG bytes, written next to the output
/// when `--image-output-dir` is set).
Expand Down Expand Up @@ -125,6 +128,9 @@ struct BatchParseCommand {
quiet: bool,
#[arg(long)]
num_workers: Option<usize>,
/// Continue parsing when OCR fails and report failed pages in JSON output.
#[arg(long)]
ocr_failure_non_fatal: bool,
/// Include per-page complexity signals as a `complexity` object on each
/// page of JSON output. Off by default.
#[arg(long)]
Expand Down Expand Up @@ -189,6 +195,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
quiet: cmd.quiet,
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
ocr_failure_fatal: !cmd.ocr_failure_non_fatal,
image_mode,
extract_links: !cmd.no_links,
include_complexity: cmd.complexity,
Expand All @@ -200,7 +207,11 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
let lp = LiteParse::new(config);
let result = rt.block_on(lp.parse(&cmd.file))?;
let formatted = match lp.config().output_format {
OutputFormat::Json => json::format_json(&result.pages)?,
OutputFormat::Json => json::format_json(
&result.pages,
&result.failed_ocr_pages,
&result.ocr_failures,
)?,
OutputFormat::Text => text::format_text(&result.pages),
OutputFormat::Markdown => result.text.clone(),
};
Expand Down Expand Up @@ -287,6 +298,7 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
quiet: cmd.quiet,
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
ocr_failure_fatal: !cmd.ocr_failure_non_fatal,
include_complexity: cmd.complexity,
..Default::default()
};
Expand Down Expand Up @@ -328,9 +340,12 @@ pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
Ok(result) => {
let fmt_result: Result<String, Box<dyn std::error::Error>> =
match lp.config().output_format {
OutputFormat::Json => {
json::format_json(&result.pages).map_err(|e| e.into())
}
OutputFormat::Json => json::format_json(
&result.pages,
&result.failed_ocr_pages,
&result.ocr_failures,
)
.map_err(|e| e.into()),
OutputFormat::Text => Ok(text::format_text(&result.pages)),
OutputFormat::Markdown => Ok(result.text.clone()),
};
Expand Down
36 changes: 35 additions & 1 deletion crates/liteparse-python/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,9 +180,32 @@ struct PyParseResult {
#[pyo3(get)]
text: String,
#[pyo3(get)]
failed_ocr_pages: Vec<usize>,
#[pyo3(get)]
ocr_failures: Vec<PyOcrFailure>,
#[pyo3(get)]
images: Vec<PyExtractedImage>,
}

#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyOcrFailure {
#[pyo3(get)]
page_number: usize,
#[pyo3(get)]
error: String,
}

#[pymethods]
impl PyOcrFailure {
fn __repr__(&self) -> String {
format!(
"OcrFailure(page_number={}, error={:?})",
self.page_number, self.error
)
}
}

#[pymethods]
impl PyParseResult {
#[getter]
Expand All @@ -196,9 +219,10 @@ impl PyParseResult {

fn __repr__(&self) -> String {
format!(
"ParseResult(pages={}, text_len={}, images={})",
"ParseResult(pages={}, text_len={}, failed_ocr_pages={}, images={})",
self.pages.len(),
self.text.len(),
self.failed_ocr_pages.len(),
self.images.len()
)
}
Expand All @@ -213,6 +237,15 @@ impl PyParseResult {
.map(PyParsedPage::from_rust)
.collect(),
text: result.text,
failed_ocr_pages: result.failed_ocr_pages,
ocr_failures: result
.ocr_failures
.into_iter()
.map(|failure| PyOcrFailure {
page_number: failure.page_number,
error: failure.error,
})
.collect(),
images: result
.images
.into_iter()
Expand Down Expand Up @@ -693,6 +726,7 @@ fn _liteparse(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<LiteParse>()?;
m.add_class::<PyLiteParseConfig>()?;
m.add_class::<PyParseResult>()?;
m.add_class::<PyOcrFailure>()?;
m.add_class::<PyExtractedImage>()?;
m.add_class::<PyParsedPage>()?;
m.add_class::<PyTextItem>()?;
Expand Down
19 changes: 19 additions & 0 deletions crates/liteparse-wasm/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,19 @@ pub struct ParsedPage {
pub struct ParseResult {
pub pages: Vec<ParsedPage>,
pub text: String,
pub failed_ocr_pages: Vec<usize>,
pub ocr_failures: Vec<OcrFailure>,
pub images: Vec<ExtractedImage>,
}

#[derive(Serialize, Tsify)]
#[tsify(into_wasm_abi)]
#[serde(rename_all = "camelCase")]
pub struct OcrFailure {
pub page_number: usize,
pub error: String,
}

#[derive(Serialize, Tsify)]
#[tsify(into_wasm_abi)]
#[serde(rename_all = "camelCase")]
Expand Down Expand Up @@ -520,6 +530,15 @@ impl LiteParse {
Ok(ParseResult {
pages,
text: result.text.clone(),
failed_ocr_pages: result.failed_ocr_pages.clone(),
ocr_failures: result
.ocr_failures
.iter()
.map(|failure| OcrFailure {
page_number: failure.page_number,
error: failure.error.clone(),
})
.collect(),
images,
})
}
Expand Down
25 changes: 21 additions & 4 deletions crates/liteparse/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ struct ParseCommand {
#[arg(long)]
num_workers: Option<usize>,

/// Continue parsing when OCR fails and report failed pages in JSON output
#[arg(long)]
ocr_failure_non_fatal: bool,

/// How to surface raster images in markdown output:
/// `off` strips them, `placeholder` (default) emits `![](image_pN_K.png)`
/// references in reading order, `embed` extracts each image's PNG bytes
Expand Down Expand Up @@ -214,6 +218,10 @@ struct BatchParseCommand {
#[arg(long)]
num_workers: Option<usize>,

/// Continue parsing when OCR fails and report failed pages in JSON output
#[arg(long)]
ocr_failure_non_fatal: bool,

/// Include per-page complexity signals as a `complexity` object on each
/// page of JSON output. Off by default.
#[arg(long)]
Expand Down Expand Up @@ -317,6 +325,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
quiet: cmd.quiet,
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
ocr_failure_fatal: !cmd.ocr_failure_non_fatal,
image_mode,
extract_links: !cmd.no_links,
include_complexity: cmd.complexity,
Expand All @@ -329,7 +338,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let lp = LiteParse::new(config);
let result = lp.parse(&cmd.file).await?;
let formatted = match lp.config().output_format {
OutputFormat::Json => json::format_json(&result.pages)?,
OutputFormat::Json => json::format_json(
&result.pages,
&result.failed_ocr_pages,
&result.ocr_failures,
)?,
OutputFormat::Text => text::format_text(&result.pages),
OutputFormat::Markdown => result.text.clone(),
};
Expand Down Expand Up @@ -420,6 +433,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
quiet: cmd.quiet,
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
ocr_failure_fatal: !cmd.ocr_failure_non_fatal,
include_complexity: cmd.complexity,
..Default::default()
};
Expand Down Expand Up @@ -464,9 +478,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(result) => {
let fmt_result: Result<String, Box<dyn std::error::Error>> =
match lp.config().output_format {
OutputFormat::Json => {
json::format_json(&result.pages).map_err(|e| e.into())
}
OutputFormat::Json => json::format_json(
&result.pages,
&result.failed_ocr_pages,
&result.ocr_failures,
)
.map_err(|e| e.into()),
OutputFormat::Text => Ok(text::format_text(&result.pages)),
OutputFormat::Markdown => Ok(result.text.clone()),
};
Expand Down
Loading
Loading