From 026144e127b40dcd9e2ded3e8e31a215bfe36c72 Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:59:57 +0200 Subject: [PATCH 01/19] M10: add multilingual OCR and automatic scan correction Recognize English, German, French, Spanish and horizontal Japanese locally. Correct quarter-turn orientation and skew before OCR, preserve scan streams, and write searchable Unicode text with undo and stale-result protection. Co-authored-by: Codex --- ATTRIBUTION.md | 5 +- ATTRIBUTION.toml | 71 ++++++++--- Cargo.lock | 10 ++ NOTICE | 4 + ROADMAP.md | 2 + crates/automation/src/a11y.rs | 22 +++- crates/automation/src/lib.rs | 1 + crates/automation/src/tools.rs | 9 +- crates/automation/tests/automation.rs | 24 ++++ crates/engine/src/catalog.rs | 2 +- crates/engine/src/commands.rs | 1 + crates/engine/src/lib.rs | 10 +- crates/engine/src/ocr.rs | 120 ++++++++++++++++-- crates/engine/src/ocr_pdf.rs | 123 ++++++++++++++++++ crates/engine/src/tests.rs | 2 +- crates/engine/tests/scan_ocr.rs | 175 ++++++++++++++++++++++++++ crates/ocr/Cargo.toml | 5 +- crates/ocr/README.md | 36 ++++++ crates/ocr/src/lib.rs | 68 +++++++--- crates/ocr/src/multilingual.rs | 142 +++++++++++++++++++++ crates/ocr/src/scan.rs | 149 ++++++++++++++++++++++ crates/ui-egui/src/commands.rs | 9 +- crates/ui-egui/src/i18n/ar.tsv | 13 ++ crates/ui-egui/src/i18n/bg.tsv | 13 ++ crates/ui-egui/src/i18n/cs.tsv | 13 ++ crates/ui-egui/src/i18n/de.tsv | 13 ++ crates/ui-egui/src/i18n/es.tsv | 13 ++ crates/ui-egui/src/i18n/fr.tsv | 13 ++ crates/ui-egui/src/i18n/hu.tsv | 13 ++ crates/ui-egui/src/i18n/it.tsv | 13 ++ crates/ui-egui/src/i18n/ja.tsv | 13 ++ crates/ui-egui/src/i18n/pl.tsv | 13 ++ crates/ui-egui/src/i18n/pt-br.tsv | 13 ++ crates/ui-egui/src/i18n/ru.tsv | 13 ++ crates/ui-egui/src/i18n/te.tsv | 13 ++ crates/ui-egui/src/i18n/uk.tsv | 13 ++ crates/ui-egui/src/i18n/zh-hans.tsv | 13 ++ crates/ui-egui/src/i18n/zh-hant.tsv | 13 ++ crates/ui-egui/src/ocr_ui.rs | 74 +++++++++-- crates/ui-egui/tests/ocr.rs | 21 ++++ docs/gaps.md | 22 ++-- docs/roadmap.md | 6 +- parity/acrobat-features.toml | 34 ++--- 43 files changed, 1258 insertions(+), 97 deletions(-) create mode 100644 crates/engine/src/ocr_pdf.rs create mode 100644 crates/engine/tests/scan_ocr.rs create mode 100644 crates/ocr/README.md create mode 100644 crates/ocr/src/multilingual.rs create mode 100644 crates/ocr/src/scan.rs diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 28bd17c7c8..2772c9fea5 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -284,7 +284,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `hayro-cmap` 0.1.0 | `assets/cmaps.brotli` | Adobe CMap resources (predefined CJK CMaps) (non-visual data, AGENTS.md §1.1) | Adobe | BSD-3-Clause | https://github.com/adobe-type-tools/cmap-resources (via hayro-cmap) | Character-code mapping tables required to read CJK PDFs. Non-visual data, allowed by AGENTS.md §1.1 | | `hayro-interpret` 0.7.0 | `src/font/generated/metrics.rs` | Standard-14 font metrics and encodings (as Rust tables) (non-visual data, AGENTS.md §1.1) | Adobe (Core 14 AFM metrics, PDF specification encodings); tables by The Hayro Authors | MIT OR Apache-2.0 | ISO 32000-2 Annex D; Adobe Core14 AFM files (via vendored hayro-interpret) | Glyph widths and encodings for non-embedded standard fonts. Non-visual data, allowed by AGENTS.md §1.1 | -## Downloaded at build time (22) +## Downloaded at build time (25) Fonts are fetched by `cargo xtask demo-pdf` into `target/demo-fonts/`, OCR models by `cargo xtask models` into `assets/models/`; each is verified by SHA-256 and never committed. @@ -312,6 +312,9 @@ Fonts are fetched by `cargo xtask demo-pdf` into `target/demo-fonts/`, OCR model | `Inter-Italic[opsz,wght].ttf` | Inter Italic (variable) | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/google/fonts/tree/9710da1eacb3be272583c3224dcb70f9da6eadbb/ofl/inter | Showcase PDF (cargo xtask demo-pdf) | | `text-detection.rten` | Ocrs text detection model | Robert Knight (ocrs-models; trained on HierText, CC-BY-SA-4.0) | CC-BY-SA-4.0 | https://github.com/robertknight/ocrs-models (the models ocrs downloads; model card: https://huggingface.co/robertknight/ocrs, cc-by-sa-4.0) | Scan & OCR: finds words on page images (cargo xtask models) | | `text-recognition.rten` | Ocrs text recognition model | Robert Knight (ocrs-models; trained on HierText, CC-BY-SA-4.0) | CC-BY-SA-4.0 | https://github.com/robertknight/ocrs-models (the models ocrs downloads; model card: https://huggingface.co/robertknight/ocrs, cc-by-sa-4.0) | Scan & OCR: reads the words it finds (cargo xtask models) | +| `multilingual-recognition.onnx` | PP-OCRv6 small multilingual text recognition model | PaddlePaddle / PaddleOCR contributors | Apache-2.0 | https://huggingface.co/PaddlePaddle/PP-OCRv6_small_rec_onnx/resolve/b8f84f0b80c529de40b4fbb3544b84fa7233a513/inference.onnx | Offline multilingual OCR and scan orientation correction (cargo xtask models) | +| `document-orientation.onnx` | PP-LCNet document orientation model | PaddlePaddle / PaddleOCR contributors | Apache-2.0 | https://huggingface.co/PaddlePaddle/PP-LCNet_x1_0_doc_ori_onnx/resolve/7330ab7039123e46af2dc03154b9969aa412c61d/inference.onnx | Offline multilingual OCR and scan orientation correction (cargo xtask models) | +| `multilingual-dictionary.txt` | PP-OCRv6 recognition dictionary | PaddlePaddle / PaddleOCR contributors | Apache-2.0 | https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf/ppocr/utils/dict/ppocrv6_dict.txt | Offline multilingual OCR and scan orientation correction (cargo xtask models) | ## Optional build inputs (1) diff --git a/ATTRIBUTION.toml b/ATTRIBUTION.toml index 17a90044da..cbcee1b588 100644 --- a/ATTRIBUTION.toml +++ b/ATTRIBUTION.toml @@ -19,7 +19,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Simplified Chinese UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "500ba36d4a37162d935a1b5e2b16eff5b2011c6aa9f9285a87d46807acdb97e6" +sha256 = "bc18c694e5ba75cdd735ff9d5c7e6dfcdc9c6e3d299ce7098c0e6be1a74f5539" [[asset]] path = "crates/ui-egui/src/i18n/zh-hant.tsv" @@ -30,7 +30,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Traditional Chinese UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "57c35909b2f19f16ff0497d110abbf1c79a5627d2a98f95706e726c59348d09e" +sha256 = "ef32a3b593d78368787dfeb1676c4f3baf82aaa7d2594d493947a36a9c671352" [[asset]] path = "crates/ui-egui/src/i18n/cs.tsv" @@ -41,7 +41,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Czech UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "abd8d0c409be846fcf48cbc1338def7f77f2d301bd84d863e4ad66434e5f821f" +sha256 = "e9a2888568dd1b528b59c1a4d03fdbef11256a0ea64d894c8e8ee2ac22c4ab81" [[asset]] path = "crates/ui-egui/src/i18n/pt-br.tsv" @@ -52,7 +52,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Brazilian Portuguese UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "130059ef7014f1dfb59ec8dc2435657d5158abca2d1825959c6366288d28ea92" +sha256 = "0a27515f2bcf8550ed13525b15a6e3b6e5532db0bcee4ec34839288c38c59542" [[asset]] path = "crates/ui-egui/src/i18n/es.tsv" @@ -63,7 +63,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Spanish UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "c67b49f2caa527fe5a241b1046f582b83f324ac86945319fd489971c038acd9b" +sha256 = "faaca01faa157ac68e097d3fbe18517cd10fd3b4a086f7d7a930a90eccc49ccd" [[asset]] path = "crates/ui-egui/src/i18n/fr.tsv" @@ -74,7 +74,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "French UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "4e8e7f71dd896c58ad2bcad8d8c57ae8c9933611cff412985eb32845821f8899" +sha256 = "19bfd863ae076e4dd8b26d2e7b345d13ff5eeac3469974d416dc0a61313ef07a" [[asset]] path = "crates/ui-egui/src/i18n/de.tsv" @@ -85,7 +85,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "German UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "09db52c316a7e9312a30d80ce4e385f2c957096563c509b7b6ed2bcc95bf39d8" +sha256 = "d0317efbb571741fac6b40782176875d566c065deb3bcd9693382299e18f1598" [[asset]] path = "crates/ui-egui/src/i18n/te.tsv" @@ -96,7 +96,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Telugu UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "21767432503afac87196a87557637692dcec76c0eba02a1dcf6bd1294926107a" +sha256 = "bd4ae6a4ef7a93e14034c6c2fd08be14d8855b0ab7171a505b58f55d27c72683" [[asset]] path = "crates/ui-egui/src/i18n/it.tsv" @@ -107,7 +107,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Italian UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "58f9514a06c74a7cdf90c6f7fb86ebb092af39857633f0779b44f0867a1c96e0" +sha256 = "d080b9e71786b8b60443091dccd51896cad86682d826e859126e88f18de98600" [[asset]] path = "crates/ui-egui/src/i18n/pl.tsv" @@ -118,7 +118,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Polish UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "7052f5ee37d26e0173bc5fe48411c94e1b48e31e0fc8a9188fca2bec54f29e32" +sha256 = "96bfd0184f11e46d4b71dd5878b440d685b2777a3f7d0a0754f3a3e24c9fc790" [[asset]] path = "crates/ui-egui/src/i18n/ru.tsv" @@ -129,7 +129,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Russian UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "e780ffd93f9606c1a762862a961c3a82bde948a8a5f116b46e6f78ba84ac6ff4" +sha256 = "7ed52410342a059cd313bccfb1b403fd6dbaa08a2723afc86bbec6c37d871138" [[asset]] path = "crates/ui-egui/src/i18n/hu.tsv" @@ -140,7 +140,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Hungarian UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "ebbd6cd7bc586ca253047f7aa3b5c34cd9ecb29645afc3d9d2097c35277ffddc" +sha256 = "87c5c7edfdea25d137354067b7ce54bdffc6f31d54ca101fb1176873c3a642e2" [[asset]] path = "crates/ui-egui/src/i18n/bg.tsv" @@ -151,7 +151,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Bulgarian UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "b760879b720c1d6de0dc9286ab044bebe4360962dece93060d7041b8955be7b2" +sha256 = "073393934129f8dc34cfdda64bb8f6c216bab7f5b2eaeb39e1387e567488cce2" [[asset]] path = "crates/ui-egui/src/i18n/ja.tsv" @@ -162,7 +162,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Japanese UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "b07366d224bd5a394f705c17fa5d26e4f45d8903927e5dc18ed12e7713751d0a" +sha256 = "a78740a318220ab0b5ca1eb2801de62515154ba66875fe1079b07dbe0e3f04b4" [[asset]] path = "crates/ui-egui/src/i18n/uk.tsv" @@ -173,7 +173,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Ukrainian UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "97c79be066c982347b48556a19b71e0db67a7f46932d1ce342b2ba5d29bf4582" +sha256 = "2405f94045bd321fc205ff206df65da647037a060c2579b0b343f14ebce67a7f" [[asset]] path = "crates/ui-egui/src/i18n/ar.tsv" @@ -184,7 +184,7 @@ licence = "MIT OR Apache-2.0" licence_file = "LICENSE-MIT" kind = "translation" usage = "Arabic UI strings, compiled in by crates/ui-egui/src/i18n/mod.rs" -sha256 = "f5e369934bf2ceca49224bf57f72289f0c82fc4c81b7c6363676d39362750d17" +sha256 = "ae9209c4eabe9726e9c4851f07fead744bfd4ac427a60fc8b4d72da133b91d4f" [[asset]] path = "assets/icons/accessibility.svg" @@ -3285,3 +3285,42 @@ licence_file = "LICENSE-MIT" kind = "document" usage = "Signature validation tests (crates/sign/tests/pdf.rs): the legacy adbe.x509.rsa_sha1 format" sha256 = "b77bfb21166f4bd043597e45a1ea28b9d9f4ba6df0142b3a0fc8b7f8e0fc111b" + +[[fetched]] +file = "multilingual-recognition.onnx" +url = "https://huggingface.co/PaddlePaddle/PP-OCRv6_small_rec_onnx/resolve/b8f84f0b80c529de40b4fbb3544b84fa7233a513/inference.onnx" +title = "PP-OCRv6 small multilingual text recognition model" +author = "PaddlePaddle / PaddleOCR contributors" +source = "https://huggingface.co/PaddlePaddle/PP-OCRv6_small_rec_onnx/resolve/b8f84f0b80c529de40b4fbb3544b84fa7233a513/inference.onnx" +licence = "Apache-2.0" +licence_url = "https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf/LICENSE" +licence_sha256 = "3840c5c0c61c294264d2dd77b8777be6ddd90121ef4e0e64abcd22edea581d6e" +kind = "model" +usage = "Offline multilingual OCR and scan orientation correction (cargo xtask models)" +sha256 = "5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634" + +[[fetched]] +file = "document-orientation.onnx" +url = "https://huggingface.co/PaddlePaddle/PP-LCNet_x1_0_doc_ori_onnx/resolve/7330ab7039123e46af2dc03154b9969aa412c61d/inference.onnx" +title = "PP-LCNet document orientation model" +author = "PaddlePaddle / PaddleOCR contributors" +source = "https://huggingface.co/PaddlePaddle/PP-LCNet_x1_0_doc_ori_onnx/resolve/7330ab7039123e46af2dc03154b9969aa412c61d/inference.onnx" +licence = "Apache-2.0" +licence_url = "https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf/LICENSE" +licence_sha256 = "3840c5c0c61c294264d2dd77b8777be6ddd90121ef4e0e64abcd22edea581d6e" +kind = "model" +usage = "Offline multilingual OCR and scan orientation correction (cargo xtask models)" +sha256 = "af9a0a4f317ff0709ce752067807f819cb15d883f8ecad89f28df1c6ee2d9c92" + +[[fetched]] +file = "multilingual-dictionary.txt" +url = "https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf/ppocr/utils/dict/ppocrv6_dict.txt" +title = "PP-OCRv6 recognition dictionary" +author = "PaddlePaddle / PaddleOCR contributors" +source = "https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf/ppocr/utils/dict/ppocrv6_dict.txt" +licence = "Apache-2.0" +licence_url = "https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf/LICENSE" +licence_sha256 = "3840c5c0c61c294264d2dd77b8777be6ddd90121ef4e0e64abcd22edea581d6e" +kind = "model" +usage = "Offline multilingual OCR and scan orientation correction (cargo xtask models)" +sha256 = "b5f2bfe2bdd9448429e3e82b51c789775d9b42f2403d082b00662eb77e401c5d" diff --git a/Cargo.lock b/Cargo.lock index d42992b9ce..ef1138665a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3792,9 +3792,12 @@ dependencies = [ name = "pdfcraft-ocr" version = "0.6.0" dependencies = [ + "image", "ocrs", "pdfcraft-fonts", "rten", + "rten-imageproc", + "rten-tensor", "thiserror 2.0.21", ] @@ -4738,6 +4741,7 @@ dependencies = [ "rten-base", "rten-gemm", "rten-model-file", + "rten-onnx", "rten-parallel", "rten-shape-inference", "rten-simd", @@ -4787,6 +4791,12 @@ dependencies = [ "rten-base", ] +[[package]] +name = "rten-onnx" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96944374568990b8d1bbc5bae5957585866f9dd8256130460405730948bfd42e" + [[package]] name = "rten-parallel" version = "0.26.0" diff --git a/NOTICE b/NOTICE index 7218477937..6da349f870 100644 --- a/NOTICE +++ b/NOTICE @@ -56,3 +56,7 @@ dependencies change (plan/adr/0002, 0006). Vendored (vendor/, patched; see vendor/README.md): - hayro 0.7.1 — Apache-2.0 OR MIT, Copyright the hayro authors (licence files kept in vendor/hayro). - hayro-interpret 0.7.0 — Apache-2.0 OR MIT, Copyright the hayro authors (LICENSE-APACHE, LICENSE-MIT kept in vendor/hayro-interpret). + +PaddleOCR PP-OCRv6 and PP-LCNet model weights and recognition dictionary +Copyright PaddlePaddle / PaddleOCR contributors. Licensed under Apache-2.0. +Fetched from the authors' pinned releases; see ATTRIBUTION.toml. diff --git a/ROADMAP.md b/ROADMAP.md index 94ca375d01..3c294ece7f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -133,6 +133,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-11 (M10, scan correction):** Offline OCR adds German, French, Spanish and horizontal Japanese with Unicode search/copy, confidence-gated quarter-turn correction, and deskew up to 12 degrees. Correction can run without OCR through the UI and automation; original image streams survive, changes undo together, and stale results are refused. Deskew on annotated pages, vertical Japanese, dewarping and editable-text output remain open. Overall percentage unchanged. + Newest first: the date, what moved, and the new overall estimate. - **2026-10-10 (M14, interface size):** Preferences ▸ Interface size scales the whole interface (50–250 %, setting `ui_scale`, control option `ui-scale`); 50 % brings an interface the system draws at 200 % back to 100 % (#456). Auto, the default, follows the desktop's text scaling: on Linux and the BSDs the desktop app reads GNOME's `text-scaling-factor` once at launch, so with Large Text (1.25) on a 1× display the interface no longer draws 13 px labels that show their pixels next to the system's text. The size is egui's zoom factor, so `pixels_per_point` grows with it and page rasters stay one texel per screen pixel; egui's own ⌘+/⌘− interface zoom is turned off so those keys stay page zoom. Still missing: KDE's font DPI and Windows' "Make text bigger" aren't read (both scale only text there), and egui's text is still greyscale-antialiased without the system's gamma, so small text looks rougher than GTK's at 100 %. diff --git a/crates/automation/src/a11y.rs b/crates/automation/src/a11y.rs index 81f79d14ee..16b7747994 100644 --- a/crates/automation/src/a11y.rs +++ b/crates/automation/src/a11y.rs @@ -150,6 +150,15 @@ impl Automation { if let Some(s) = a.opt_bool("skip_text_pages")? { settings.skip_text_pages = s; } + if let Some(v) = a.opt_bool("auto_rotate")? { + settings.auto_rotate = v; + } + if let Some(v) = a.opt_bool("deskew")? { + settings.deskew = v; + } + if let Some(v) = a.opt_bool("recognize")? { + settings.recognize = v; + } Ok(settings) } @@ -185,7 +194,16 @@ impl Automation { } pub(crate) fn ocr_recognize(&mut self, a: &Args) -> Result { - let settings = self.ocr_settings(a)?; + self.ocr_process(a, false) + } + pub(crate) fn ocr_enhance(&mut self, a: &Args) -> Result { + self.ocr_process(a, true) + } + fn ocr_process(&mut self, a: &Args, correction_only: bool) -> Result { + let mut settings = self.ocr_settings(a)?; + if correction_only { + settings.recognize = false; + } let pages = if a.opt_ints("pages")?.is_some() { self.pages(a, "pages")? } else { Vec::new() }; let id = self.doc(a)?.id; let found = self.session.recognize_text(id, &pages, settings).map_err(failed)?; @@ -193,7 +211,7 @@ impl Automation { .iter() .map(|p| match &p.skipped { Some(why) => json!({ "page": p.page + 1, "skipped": why }), - None => json!({ "page": p.page + 1, "words": p.words.len(), "text": p.text() }), + None => json!({ "page": p.page + 1, "words": p.words.len(), "text": p.text(), "rotation":p.correction.rotation,"deskew":p.correction.deskew }), }) .collect(); Ok(json!({ "words": found.iter().map(|p| p.words.len()).sum::(), "pages": list })) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index eec1f43606..b856e978ef 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -570,6 +570,7 @@ impl Automation { "sign_list" => self.sign_list(&a)?, "accessibility_check" => self.a11y_check(&a)?, "ocr_recognize" => self.ocr_recognize(&a)?, + "ocr_enhance" => self.ocr_enhance(&a)?, "js_run" => self.js_run(&a)?, "js_document_scripts" => self.js_document_scripts(&a)?, "js_set_document_script" => self.js_set_document_script(&a)?, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index f5771d68c9..8489a5e314 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -917,14 +917,14 @@ pub fn tools() -> Vec { )), t("js_enabled", "JavaScript on or off", "Preferences ▸ JavaScript ▸ Enable Acrobat JavaScript: set it with `enabled`, or read it. With JavaScript off, field scripts other than Acrobat's AF calls don't run.") .with(schema(json!({ "enabled": { "type": "boolean" } }), &[])), - t("ocr_recognize", "Recognize text (OCR)", "Scan & OCR ▸ Recognize text: render pages, read the words in them and add them as invisible text over the page image, so scanned pages become searchable and selectable (a searchable image; the image is not changed). Pages that already have text are skipped unless skip_text_pages is false. Returns each page's recognised text, word count or why it was skipped. Needs the OCR models (ocr_status). Undoable as one step.") + t("ocr_recognize", "Recognize text (OCR)", "Scan & OCR ▸ Recognize text: render pages, read the words in them and add them as invisible text over the page image, so scanned pages become searchable and selectable (a searchable image). Automatically corrects orientation and skew by default. Set auto_rotate and deskew to false for an exact image, or recognize to false for correction only. Pages that already have text are skipped unless skip_text_pages is false. Returns each page's recognised text, word count or why it was skipped. Needs the OCR models (ocr_status). Undoable as one step.") .cmd("ocr.recognize") .with(schema( json!({ "doc": doc(), "pages": pages("to recognise (default: all)"), "dpi": { "type": "number", "minimum": 72, "maximum": 600, "description": "Resolution pages are read at (default 300; a scanned page is read at most at its own resolution)." }, - "language": { "type": "string", "enum": ["en"], "description": "Document language (default en)." }, + "auto_rotate": { "type": "boolean", "default": true }, "deskew": { "type": "boolean", "default": true }, "recognize": { "type": "boolean", "default": true }, "language": { "type": "string", "enum": ["en", "de", "ja", "fr", "es"], "description": "Document language (default en)." }, "skip_text_pages": { "type": "boolean", "description": "Leave pages that already contain text alone (default true)." }, }), &["doc"], @@ -936,10 +936,13 @@ pub fn tools() -> Vec { "paths": { "type": "array", "items": { "type": "string" }, "minItems": 1 }, "folder": { "type": "string" }, "dpi": { "type": "number", "minimum": 72, "maximum": 600 }, - "language": { "type": "string", "enum": ["en"] }, + "auto_rotate": { "type": "boolean", "default": true }, "deskew": { "type": "boolean", "default": true }, "recognize": { "type": "boolean", "default": true }, "language": { "type": "string", "enum": ["en", "de", "ja", "fr", "es"] }, }), &["paths", "folder"], )), + t("ocr_enhance", "Correct scanned pages", "Detect and correct sideways/upside-down scans and straighten skew up to 12 degrees without adding text. Original image data is preserved. Low-confidence rotations are left unchanged. Deskew refuses pages with annotations to avoid misalignment. Undoable as one step.") + .cmd("ocr.enhance") + .with(schema(json!({"doc":doc(),"pages":pages("to correct (default all)"),"dpi":{"type":"number","minimum":72,"maximum":600},"auto_rotate":{"type":"boolean","default":true},"deskew":{"type":"boolean","default":true},"skip_text_pages":{"type":"boolean","default":true}}), &["doc"])), t("ocr_status", "OCR status", "Whether text recognition is available (its models are installed: release packages include them; a source build fetches them with `cargo xtask models`; PDFCRAFT_MODELS overrides), where it looks for them, and the languages it reads.") .ro() .cmd("ocr.recognize") diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 26bc2b26f3..f4907c8796 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -2947,6 +2947,9 @@ fn ocr_tools_make_a_scan_searchable() { let mut a = auto(&dir); let status = ok(&mut a, "ocr_status", json!({})); assert_eq!(status["languages"][0]["code"], "en"); + for language in ["de", "ja", "fr", "es"] { + assert!(status["languages"].as_array().unwrap().iter().any(|l| l["code"] == language)); + } if status["available"] != true { eprintln!("skipped: OCR models not installed"); return; @@ -2963,6 +2966,27 @@ fn ocr_tools_make_a_scan_searchable() { assert!(matches!(a.call("ocr_recognize", &json!({ "doc": scan, "language": "xx" })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn ocr_enhance_corrects_rotation_without_adding_text() { + let dir = workdir("ocr-enhance"); + let mut a = auto(&dir); + if ok(&mut a, "ocr_status", json!({}))["available"] != true { + return; + } + let sentence = + (0..18).map(|_| "Please read this document with the page upright. The quick brown fox jumps over the lazy dog.\n").collect::(); + let doc = ok(&mut a, "doc_create", json!({"from":"text","text":sentence}))["doc"].as_u64().unwrap(); + ok(&mut a, "page_rotate", json!({"doc":doc,"pages":[1],"degrees":180})); + ok(&mut a, "doc_export_images", json!({"doc":doc,"folder":"scan","dpi":150,"pages":[1]})); + let png = std::fs::read_dir(dir.join("scan")).unwrap().next().unwrap().unwrap().path(); + let scan = ok(&mut a, "doc_create", json!({"from":"images","paths":[png.to_string_lossy()]}))["doc"].as_u64().unwrap(); + let corrected = ok(&mut a, "ocr_enhance", json!({"doc":scan,"deskew":false})); + assert_eq!(corrected["pages"][0]["rotation"], 180, "{corrected}"); + assert_eq!(corrected["words"], 0); + let read = ok(&mut a, "ocr_recognize", json!({"doc":scan,"auto_rotate":false,"deskew":false})); + assert!(read["pages"][0]["text"].as_str().unwrap().contains("Please"), "{read}"); +} + #[test] fn ocr_recognize_files_refuses_paths_outside_the_root() { let dir = workdir("ocr-files-root"); diff --git a/crates/engine/src/catalog.rs b/crates/engine/src/catalog.rs index c553675cd1..2fb364dd0b 100644 --- a/crates/engine/src/catalog.rs +++ b/crates/engine/src/catalog.rs @@ -249,7 +249,7 @@ pub static TOOL_GROUPS: &[ToolGroup] = &[ items: &[ item("In this file", "scan-text", "ocr.recognize", Ready), item("In multiple files", "files", "ocr.recognize_batch", Ready), - item("Enhance scanned file", "sparkles", "ocr.enhance", Planned("M10")), + item("Enhance scanned file", "sparkles", "ocr.enhance", Ready), item("Correct recognized text", "text-select", "ocr.correct", Planned("M10")), ], }], diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index 1f68eeb1f2..8cbe2bbea2 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -309,6 +309,7 @@ pub const COMMANDS: &[CommandSpec] = &[ c("tools.js_console", "JavaScript console…", None, Some(Shortcut::cmd("J")), Document, "square-terminal"), c("tools.document_js", "Document JavaScripts…", None, None, Modification, "file-code"), c("ocr.recognize", "Recognize text…", None, None, Modification, "scan-text"), + c("ocr.enhance", "Enhance scanned file", None, None, Modification, "sparkles"), c("ocr.recognize_batch", "Recognize text in multiple files…", None, None, Nothing, "files"), c("a11y.check", "Check for accessibility…", None, None, Document, "accessibility"), c("a11y.report", "Open accessibility report", None, None, Document, "file-text"), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 1c059a22a1..278da22eb5 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -21,6 +21,7 @@ pub mod export; pub mod js; pub mod links; pub mod ocr; +mod ocr_pdf; pub mod optimizer; pub mod signature_image; mod upright_image; @@ -1050,6 +1051,10 @@ pub enum Edit { }, /// Scan & OCR ▸ Recognize text: put recognised words on `page` as invisible text over the /// image (from [`ocr::OcrJob::run`]). + DeskewPage { + page: usize, + degrees: f64, + }, AddOcrText { page: usize, words: Vec, @@ -1255,6 +1260,7 @@ impl Edit { Edit::SetInitialView(_) => "Change initial view".into(), Edit::SetAltText { .. } => "Set alternate text".into(), Edit::MarkDecorative { .. } => "Mark figure as decorative".into(), + Edit::DeskewPage { .. } => "Straighten scanned page".into(), Edit::AddOcrText { .. } => "Recognize text".into(), Edit::ApplyScriptChanges { .. } => "Run JavaScript".into(), Edit::XfaEvent { .. } => "Run form script".into(), @@ -1436,6 +1442,7 @@ fn check_permission(edit: &Edit, p: &pdfcraft_cos::Permissions) -> Result<(), Ed | Edit::SetInitialView(_) | Edit::SetAltText { .. } | Edit::MarkDecorative { .. } + | Edit::DeskewPage { .. } | Edit::AddOcrText { .. } | Edit::SetDocumentScript { .. } | Edit::SetFieldScript { .. } @@ -1691,8 +1698,9 @@ fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut EditCtx) -> pdfcraft_preflight::convert(doc, *level).map_err(|e| EditError::Invalid(e.to_string()))?; } Edit::SetDocumentScript { name, script } => pdfcraft_forms::set_document_script(doc, name, script.as_deref())?, + Edit::DeskewPage { page, degrees } => ocr_pdf::deskew(doc, *page, *degrees)?, Edit::AddOcrText { page, words } => { - pdfcraft_edit::stamp(doc, *page, "OCR", pdfcraft_ocr::text_layer(words))?; + ocr_pdf::add_text(doc, *page, words)?; } Edit::EditPageImage { page, index, change } => { let img = pdfcraft_edit::page_images(doc, *page)? diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs index cb245f14f7..cadf810b42 100644 --- a/crates/engine/src/ocr.rs +++ b/crates/engine/src/ocr.rs @@ -9,6 +9,7 @@ use std::sync::{Arc, Mutex}; use pdfcraft_render::{PageInfo, PageRenderer, RenderConfig, RenderRequest, RequestKind}; +pub use pdfcraft_ocr::scan::Correction; pub use pdfcraft_ocr::{LANGUAGES, Models, Ocr, OcrError, PlacedWord}; use crate::{DocId, Edit, EditError, Session}; @@ -18,26 +19,44 @@ use crate::{DocId, Edit, EditError, Session}; pub struct OcrSettings { /// Resolution pages are rendered at for recognition (Acrobat's "Downsample to" choices). pub dpi: f32, - /// Language code from [`LANGUAGES`] (the models read the Latin alphabet, whatever this says). + /// Language code from [`LANGUAGES`]; recognized using the multilingual model. pub language: String, /// Leave pages that already have text alone (Acrobat reports "page contains renderable /// text" and skips them). pub skip_text_pages: bool, + pub auto_rotate: bool, + pub deskew: bool, + /// Disable to straighten scans without adding a text layer. + pub recognize: bool, } impl Default for OcrSettings { fn default() -> Self { - OcrSettings { dpi: 300.0, language: "en".into(), skip_text_pages: true } + OcrSettings { dpi: 300.0, language: "en".into(), skip_text_pages: true, auto_rotate: true, deskew: true, recognize: true } } } /// What recognition found on one page. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, PartialEq)] pub struct OcrPage { pub page: usize, pub words: Vec, /// Why the page was not read, if it was skipped. pub skipped: Option, + pub correction: Correction, + /// Snapshot guard: never apply positions to pages changed while recognition was running. + source: Arc>, +} + +impl std::fmt::Debug for OcrPage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OcrPage") + .field("page", &self.page) + .field("words", &self.words) + .field("skipped", &self.skipped) + .field("correction", &self.correction) + .finish_non_exhaustive() + } } impl OcrPage { @@ -78,7 +97,28 @@ pub struct OcrJob { impl OcrJob { /// Read the pages. `progress(done, total)` is called before each page; returning `false` /// stops (the pages read so far are returned). - pub fn run(self, ocr: &Ocr, mut progress: impl FnMut(usize, usize) -> bool) -> Vec { + pub fn run(self, ocr: &Ocr, progress: impl FnMut(usize, usize) -> bool) -> Vec { + let failure = |why: &str| { + self.pages + .iter() + .map(|&page| OcrPage { + page, + words: Vec::new(), + skipped: Some(why.into()), + correction: Correction::default(), + source: self.bytes.clone(), + }) + .collect() + }; + if !self.settings.dpi.is_finite() || !(72.0..=600.0).contains(&self.settings.dpi) || !LANGUAGES.iter().any(|l| l.0 == self.settings.language) + { + return failure("invalid OCR language or resolution"); + } + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.run_inner(ocr, progress))) + .unwrap_or_else(|_| failure("scan processing failed safely; the document was not changed")) + } + + fn run_inner(&self, ocr: &Ocr, mut progress: impl FnMut(usize, usize) -> bool) -> Vec { let config = RenderConfig { password: self.password.as_deref().map(Arc::from), ..Default::default() }; let mut r = PageRenderer::new(self.bytes.clone(), config); let mut out = Vec::new(); @@ -88,7 +128,13 @@ impl OcrJob { break; } let Some(info) = self.infos.get(page) else { continue }; - let skip = |why: &str| OcrPage { page, words: Vec::new(), skipped: Some(why.into()) }; + let skip = |why: &str| OcrPage { + page, + words: Vec::new(), + skipped: Some(why.into()), + correction: Correction::default(), + source: self.bytes.clone(), + }; if self.settings.skip_text_pages { let t = r.render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }); if t.text.as_ref().is_some_and(|t| !t.plain_text().trim().is_empty()) { @@ -105,7 +151,30 @@ impl OcrJob { out.push(skip(&e)); continue; } - let lines = match ocr.recognize(&shot.rgba, shot.width, shot.height) { + let Some(rgba) = image::RgbaImage::from_raw(shot.width, shot.height, shot.rgba.to_vec()) else { + out.push(skip("invalid rendered image")); + continue; + }; + let rgb = image::DynamicImage::ImageRgba8(rgba).to_rgb8(); + let rotation = if self.settings.auto_rotate { + match ocr.orientation(&rgb) { + Ok(r) => r, + Err(e) => { + out.push(skip(&e.to_string())); + continue; + } + } + } else { + 0 + }; + let mut correction = Correction { rotation, deskew: 0.0 }; + let turned = correction.apply(&rgb); + if self.settings.deskew { + correction.deskew = pdfcraft_ocr::scan::deskew(&turned); + } + let corrected = Correction { rotation: 0, ..correction }.apply(&turned); + let result = if self.settings.recognize { ocr.recognize_language(&corrected, &self.settings.language) } else { Ok(Vec::new()) }; + let lines = match result { Ok(l) => l, Err(e) => { out.push(skip(&e.to_string())); @@ -114,11 +183,12 @@ impl OcrJob { }; let scale = shot.width as f32 / info.width.max(1e-3); let to_user = |x: f32, y: f32| { - let [u, v] = info.view_to_user(x / scale, y / scale); + let [x, y] = correction.to_source(x as f64, y as f64, shot.width as f64, shot.height as f64); + let [u, v] = info.view_to_user(x as f32 / scale, y as f32 / scale); [u as f64, v as f64] }; let words = lines.iter().flat_map(|l| &l.words).map(|w| PlacedWord::place(w, to_user)).collect(); - out.push(OcrPage { page, words, skipped: None }); + out.push(OcrPage { page, words, skipped: None, correction, source: self.bytes.clone() }); } progress(total, total); out @@ -152,17 +222,42 @@ impl Session { /// Add the words found by [`OcrJob::run`] as one undoable step. Returns the number of words. pub fn apply_ocr(&mut self, id: DocId, found: &[OcrPage]) -> Result { - let edits: Vec = - found.iter().filter(|p| !p.words.is_empty()).map(|p| Edit::AddOcrText { page: p.page, words: p.words.clone() }).collect(); + let doc = self.get(id).ok_or_else(|| EditError::Invalid("no such document".into()))?; + if found.iter().any(|p| !Arc::ptr_eq(&p.source, &doc.bytes) && p.source.as_ref() != doc.bytes.as_ref()) { + return Err(EditError::Invalid("the document changed during scan processing; run it again".into())); + } + let mut edits = Vec::new(); + for p in found.iter().filter(|p| p.skipped.is_none()) { + if !p.words.is_empty() { + edits.push(Edit::AddOcrText { page: p.page, words: p.words.clone() }); + } + if p.correction.rotation != 0 { + edits.push(Edit::RotatePages { pages: vec![p.page], degrees: i64::from(p.correction.rotation) }); + } + if p.correction.deskew.abs() >= 0.05 { + edits.push(Edit::DeskewPage { page: p.page, degrees: p.correction.deskew }); + } + } let words = found.iter().map(|p| p.words.len()).sum(); if !edits.is_empty() { - self.apply(id, Edit::Batch { label: "Recognize text".into(), edits })?; + self.apply(id, Edit::Batch { label: if words > 0 { "Recognize text" } else { "Straighten scanned pages" }.into(), edits })?; } Ok(words) } /// Recognise text on `pages` and add it, in one call (the CLI and agents use this). pub fn recognize_text(&mut self, id: DocId, pages: &[usize], settings: OcrSettings) -> Result, String> { + if !settings.dpi.is_finite() || !(72.0..=600.0).contains(&settings.dpi) { + return Err("OCR resolution must be between 72 and 600 dpi".into()); + } + if !LANGUAGES.iter().any(|l| l.0 == settings.language) { + return Err(format!("unsupported language: {}", settings.language)); + } + if let Some(doc) = self.get(id) + && pages.iter().any(|&p| p >= doc.info.pages.len()) + { + return Err("OCR page does not exist".into()); + } let job = self.ocr_job(id, pages, settings).ok_or("no such document")?; if let Some(why) = self.get(id).and_then(|d| d.read_only_reason.clone()) { return Err(why); @@ -205,7 +300,8 @@ pub fn recognize_file( } let job = s.ocr_job(id, &[], settings).ok_or("the document could not be read")?; let pages = job.run(ocr, progress); - if s.apply_ocr(id, &pages).map_err(|e| e.to_string())? == 0 { + let words = s.apply_ocr(id, &pages).map_err(|e| e.to_string())?; + if words == 0 && !pages.iter().any(|p| p.correction.changed() && p.skipped.is_none()) { return Ok(FileResult { bytes, pages }); } Ok(FileResult { bytes: s.save_bytes(id).map_err(|e| e.to_string())?, pages }) diff --git a/crates/engine/src/ocr_pdf.rs b/crates/engine/src/ocr_pdf.rs new file mode 100644 index 0000000000..309b4d7b87 --- /dev/null +++ b/crates/engine/src/ocr_pdf.rs @@ -0,0 +1,123 @@ +//! Lossless scan correction and Unicode-only invisible OCR fonts (no font assets). +use crate::EditError; +use pdfcraft_cos::{Dict, Document, Object, Stream}; +use pdfcraft_ocr::PlacedWord; +use std::collections::{BTreeMap, BTreeSet}; + +fn numbers(values: impl IntoIterator) -> Object { + Object::Array(values.into_iter().map(Object::Real).collect()) +} + +pub(crate) fn add_text(doc: &mut Document, page: usize, words: &[PlacedWord]) -> Result<(), EditError> { + let p = pdfcraft_model::pages(doc).get(page).cloned().ok_or_else(|| EditError::Invalid("no such page".into()))?; + if words.iter().any(|w| w.text.len() > 100_000 || w.origin.iter().chain(&w.across).chain(&w.up).any(|v| !v.is_finite())) { + return Err(EditError::Invalid("invalid OCR word geometry or text".into())); + } + let chars: Vec = words.iter().flat_map(|w| w.text.chars()).collect::>().into_iter().collect(); + let mut res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let mut fonts = res.get(b"Font").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let mut codes = BTreeMap::new(); + // Empty Type 3 glyphs with explicit metrics and ToUnicode keep text selectable without + // shipping any typeface. Chunks also handle pages with more than 255 unique characters. + for (index, chunk) in chars.chunks(255).enumerate() { + let mut name = format!("PCOcr{index}"); + while fonts.contains(name.as_bytes()) { + name.push('_'); + } + let glyph = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"1000 0 0 -210 1000 720 d1\n".to_vec()))); + let mut procs = Dict::new(); + let mut differences = vec![Object::Int(1)]; + let mut cmap = String::from( + "/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CIDSystemInfo << /Registry (PdfCraft) /Ordering (Unicode) /Supplement 0 >> def\n/CMapName /PdfCraftOCR def\n/CMapType 2 def\n1 begincodespacerange\n<01> \nendcodespacerange\n", + ); + for entries in chunk.chunks(100) { + cmap.push_str(&format!("{} beginbfchar\n", entries.len())); + for ch in entries { + let code = codes.len() % 255 + 1; + let g = format!("g{code}"); + procs.set(g.as_bytes().to_vec(), Object::Ref(glyph)); + differences.push(Object::name(&g)); + let mut utf16 = [0u16; 2]; + let hex: String = ch.encode_utf16(&mut utf16).iter().map(|u| format!("{u:04X}")).collect(); + cmap.push_str(&format!("<{code:02X}> <{hex}>\n")); + codes.insert(*ch, (name.clone(), code)); + } + cmap.push_str("endbfchar\n"); + } + cmap.push_str("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n"); + let cmap = doc.add(Object::Stream(Stream::flate(Dict::new(), cmap.as_bytes()))); + let mut encoding = Dict::new(); + encoding.set(b"Type".to_vec(), Object::name("Encoding")); + encoding.set(b"Differences".to_vec(), Object::Array(differences)); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type3")); + font.set(b"FontBBox".to_vec(), numbers([0.0, -210.0, 1000.0, 720.0])); + font.set(b"FontMatrix".to_vec(), numbers([0.001, 0.0, 0.0, 0.001, 0.0, 0.0])); + font.set(b"FirstChar".to_vec(), Object::Int(1)); + font.set(b"LastChar".to_vec(), Object::Int(chunk.len() as i64)); + font.set(b"Widths".to_vec(), numbers(std::iter::repeat_n(1000.0, chunk.len()))); + font.set(b"Encoding".to_vec(), Object::Dict(encoding)); + font.set(b"CharProcs".to_vec(), Object::Dict(procs)); + font.set(b"ToUnicode".to_vec(), Object::Ref(cmap)); + font.set(b"Resources".to_vec(), Object::Dict(Dict::new())); + fonts.set(name.into_bytes(), Object::Ref(doc.add(Object::Dict(font)))); + } + res.set(b"Font".to_vec(), Object::Dict(fonts)); + doc.update_dict(p.obj, |d| d.set(b"Resources".to_vec(), Object::Dict(res))).map_err(|e| EditError::Invalid(e.to_string()))?; + let mut out = String::from("/OCR BMC\nBT\n3 Tr\n"); + for word in words { + let len = word.text.chars().count() as f64; + if len == 0.0 { + continue; + } + let [a, b] = word.across.map(|v| v / len); + let [c, d] = word.up.map(|v| v / 0.93); + let (x, y) = (word.origin[0] + c * 0.21, word.origin[1] + d * 0.21); + out.push_str(&format!("{a:.6} {b:.6} {c:.6} {d:.6} {x:.6} {y:.6} Tm\n")); + for ch in word.text.chars() { + if let Some((name, code)) = codes.get(&ch) { + out.push_str(&format!("/{name} 1 Tf <{code:02X}> Tj\n")); + } + } + } + out.push_str("ET\nEMC\n"); + pdfcraft_edit::stamp(doc, page, "OCR", out.into_bytes())?; + Ok(()) +} + +pub(crate) fn deskew(doc: &mut Document, page: usize, degrees: f64) -> Result<(), EditError> { + if !degrees.is_finite() || degrees.abs() > 12.0 { + return Err(EditError::Invalid("deskew must be between -12 and 12 degrees".into())); + } + let p = pdfcraft_model::pages(doc).get(page).cloned().ok_or_else(|| EditError::Invalid("no such page".into()))?; + if p.dict.get(b"Annots").map(|v| doc.resolve(v)).and_then(|v| v.as_array().map(|a| !a.is_empty())).unwrap_or(false) { + return Err(EditError::Invalid("deskew is not available on pages with annotations; disable deskew or flatten annotations first".into())); + } + let [l, b, r, t] = p.crop(doc); + let (w, h) = (r - l, t - b); + if w <= 0.0 || h <= 0.0 { + return Err(EditError::Invalid("invalid page bounds".into())); + } + let correction = pdfcraft_ocr::scan::Correction { rotation: 0, deskew: degrees }; + let k = correction.scale(w, h); + let (s, c) = (-degrees).to_radians().sin_cos(); + let (a, bm, cm, d) = (k * c, k * s, -k * s, k * c); + let (cx, cy) = ((l + r) / 2.0, (b + t) / 2.0); + let (e, f) = (cx - a * cx - cm * cy, cy - bm * cx - d * cy); + let open = format!("q\n{a:.9} {bm:.9} {cm:.9} {d:.9} {e:.9} {f:.9} cm\n{l} {b} {w} {h} re W n\n"); + let open = Object::Ref(doc.add(Object::Stream(Stream::flate(Dict::new(), open.as_bytes())))); + let close = Object::Ref(doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"Q\n".to_vec())))); + let original = p.dict.get(b"Contents").cloned(); + let mut streams = vec![open]; + if let Some(content) = original { + if let Some(list) = doc.resolve(&content).as_array() { + streams.extend(list.iter().cloned()); + } else { + streams.push(content); + } + } + streams.push(close); + doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), Object::Array(streams))).map_err(|e| EditError::Invalid(e.to_string()))?; + Ok(()) +} diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index eee3f7ca56..f0b32bed08 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1845,7 +1845,7 @@ fn recognize_text_makes_a_scanned_page_searchable() { assert_eq!(found.len(), 1); let text = page_texts(&s, id)[0].to_lowercase(); for w in ["quick", "brown", "fox", "lazy"] { - assert!(text.contains(w), "{text}"); + assert!(text.contains(w), "{text}; correction: {:?}", found[0].correction); } assert_eq!(s.get(id).unwrap().can_undo(), Some("Recognize text")); // The text sits over the words: "quick" is left of "lazy" and on the same line. diff --git a/crates/engine/tests/scan_ocr.rs b/crates/engine/tests/scan_ocr.rs new file mode 100644 index 0000000000..1728150e6a --- /dev/null +++ b/crates/engine/tests/scan_ocr.rs @@ -0,0 +1,175 @@ +use pdfcraft_engine::{ + Edit, Session, + export::Exporter, + ocr::{self, OcrSettings, PlacedWord}, +}; + +fn render(session: &Session, id: pdfcraft_engine::DocId) -> Vec { + Exporter::new(session.get(id).unwrap()).png(0, 150.0).unwrap() +} +fn text(session: &Session, id: pdfcraft_engine::DocId) -> String { + Exporter::new(session.get(id).unwrap()).text(0).unwrap() +} +fn scan(sentence: &str, rotation: u16, skew: f64) -> (Session, pdfcraft_engine::DocId) { + let mut session = Session::new(); + let japanese = sentence.chars().any(|c| ('\u{3000}'..='\u{9fff}').contains(&c)); + let pdf = session.create_from_text("Scan fixture", if japanese { "Placeholder" } else { sentence }).unwrap(); + let doc = session.open("text.pdf", None, pdf, None).unwrap(); + if japanese { + session.apply(doc, Edit::EditTextLine { page: 0, line: 0, text: sentence.into() }).unwrap(); + } + let image = image::load_from_memory(&render(&session, doc)).unwrap().to_rgb8(); + let image = pdfcraft_ocr::scan::Correction { rotation, deskew: skew }.apply(&image); + let mut png = std::io::Cursor::new(Vec::new()); + image.write_to(&mut png, image::ImageFormat::Png).unwrap(); + let pdf = session.create_from_images(&[("scan.png".into(), png.into_inner())]).unwrap(); + let id = session.open("scan.pdf", None, pdf, None).unwrap(); + (session, id) +} + +#[test] +fn unicode_ocr_survives_save_search_and_undo_without_changing_pixels() { + let (mut s, id) = scan("Invisible text layer", 0, 0.0); + let before = render(&s, id); + let words = + vec![PlacedWord { text: "Grüße 日本語 français español 😀".into(), origin: [50.0, 50.0], across: [300.0, 0.0], up: [0.0, 20.0] }]; + s.apply(id, Edit::AddOcrText { page: 0, words }).unwrap(); + assert_eq!(before, render(&s, id)); + let found = text(&s, id); + assert!(found.contains("Grüße 日本語 français español 😀"), "{found:?}"); + let saved = s.save_bytes(id).unwrap(); + let reopened = s.open("saved.pdf", None, saved, None).unwrap(); + assert!(text(&s, reopened).contains("日本語")); + s.undo(id).unwrap(); + assert!(text(&s, id).trim().is_empty()); + s.redo(id).unwrap(); + assert!(text(&s, id).contains("日本語")); +} + +#[test] +fn annotated_scan_refuses_deskew_atomically() { + let (mut s, id) = scan("Keep comments attached", 0, 0.0); + s.apply( + id, + Edit::AddAnnotation(pdfcraft_engine::NewAnnotation { + page: 0, + shape: pdfcraft_engine::Shape::Rectangle { rect: [20.0, 20.0, 100.0, 50.0] }, + style: Default::default(), + contents: "Keep this comment".into(), + author: "Test".into(), + }), + ) + .unwrap(); + let before = s.get(id).unwrap().bytes.clone(); + let result = s.apply( + id, + Edit::Batch { + label: "Correct scans".into(), + edits: vec![Edit::RotatePages { pages: vec![0], degrees: 90 }, Edit::DeskewPage { page: 0, degrees: 3.0 }], + }, + ); + assert!(result.unwrap_err().to_string().contains("annotations")); + assert_eq!(s.get(id).unwrap().bytes, before); + for degrees in [f64::NAN, f64::INFINITY, 13.0] { + assert!(s.apply(id, Edit::DeskewPage { page: 0, degrees }).is_err()); + } +} + +#[test] +fn scan_correction_handles_all_quarter_turns_and_stale_results() { + if !ocr::available() { + eprintln!("skipped: run cargo xtask models"); + return; + } + let sentence = "Document orientation and searchable text\nThe quick brown fox jumps over the lazy dog.\nEvery page should be upright and easy to read.\nPlease keep this document for your records.\nA useful example with several lines of text."; + for rotation in [0, 90, 180, 270] { + let (mut s, id) = scan(sentence, rotation, 0.0); + let before = s.get(id).unwrap().bytes.clone(); + let result = s.recognize_text(id, &[], OcrSettings { deskew: false, ..Default::default() }).unwrap(); + assert_eq!(result[0].correction.rotation, (360 - rotation) % 360, "{result:?}"); + assert!(text(&s, id).to_lowercase().contains("orientation"), "{:?}", result); + s.undo(id).unwrap(); + assert_eq!(s.get(id).unwrap().bytes, before); + s.redo(id).unwrap(); + let job = s.ocr_job(id, &[], OcrSettings { skip_text_pages: false, ..Default::default() }).unwrap(); + let found = job.run(&ocr::engine().unwrap(), |_, _| true); + s.apply(id, Edit::RotatePages { pages: vec![0], degrees: 90 }).unwrap(); + assert!(s.apply_ocr(id, &found).unwrap_err().to_string().contains("changed")); + } +} + +#[test] +fn scan_deskew_without_ocr_persists_and_is_undoable() { + if !ocr::available() { + return; + } + let sentence = (0..18).map(|_| "Straight lines make this scanned document easier to read.\n").collect::(); + for angle in [-4.0, 3.0] { + let (mut s, id) = scan(&sentence, 0, angle); + let before = s.get(id).unwrap().bytes.clone(); + let result = s.recognize_text(id, &[], OcrSettings { auto_rotate: false, recognize: false, ..Default::default() }).unwrap(); + assert!((result[0].correction.deskew + angle).abs() < 0.3, "{result:?}"); + assert!(text(&s, id).trim().is_empty()); + let saved = s.save_bytes(id).unwrap(); + assert!(saved.starts_with(before.as_ref())); + let after = s.open("after.pdf", None, saved, None).unwrap(); + let img = image::load_from_memory(&render(&s, after)).unwrap().to_rgb8(); + assert!(pdfcraft_ocr::scan::deskew(&img).abs() < 0.2); + s.undo(id).unwrap(); + assert_eq!(s.get(id).unwrap().bytes, before); + } +} + +#[test] +fn scan_recognizes_accents_in_requested_languages() { + if !ocr::available() { + return; + } + for (language, sentence, expected) in [ + ("de", "Grüße aus Österreich und schöne Grüße für alle.", "Österreich"), + ("fr", "Une journée à Paris avec un café français.", "journée"), + ("es", "El español también tiene acentos y una mañana.", "español"), + ] { + let (mut s, id) = scan(sentence, 0, 0.0); + let found = + s.recognize_text(id, &[], OcrSettings { language: language.into(), auto_rotate: false, deskew: false, ..Default::default() }).unwrap(); + assert!(text(&s, id).contains(expected), "{language}: {found:?}; extracted {:?}", text(&s, id)); + } +} + +#[test] +fn scan_recognizes_japanese_with_release_fonts() { + if !ocr::available() || pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipped: models and CRAFT_FONTS_DIR required"); + return; + } + let (mut s, id) = scan("日本語の文字を認識します。\n今日は良い天気です。", 0, 0.0); + let found = s.recognize_text(id, &[], OcrSettings { language: "ja".into(), auto_rotate: false, deskew: false, ..Default::default() }).unwrap(); + assert!(text(&s, id).contains("日本語"), "{found:?}; extracted {:?}", text(&s, id)); +} + +#[test] +fn scan_rotation_and_deskew_keep_searchable_text_aligned() { + if !ocr::available() { + return; + } + let sentence = (0..12).map(|_| "Straight searchable lines stay aligned with the scan.\n").collect::(); + let (mut s, id) = scan(&sentence, 90, 4.0); + let found = s.recognize_text(id, &[], OcrSettings::default()).unwrap(); + assert_eq!(found[0].correction.rotation, 270); + assert!((found[0].correction.deskew + 4.0).abs() < 0.3, "{found:?}"); + assert!(text(&s, id).to_lowercase().contains("searchable")); + let bytes = s.save_bytes(id).unwrap(); + let reopened = s.open("result.pdf", None, bytes, None).unwrap(); + let img = image::load_from_memory(&render(&s, reopened)).unwrap().to_rgb8(); + assert!(pdfcraft_ocr::scan::deskew(&img).abs() < 0.2); + // The extracted words must lie along horizontal lines after both PDF transformations. + let doc = s.get(reopened).unwrap(); + let mut renderer = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), doc.export_source().config); + let text = + renderer.render(pdfcraft_render::RenderRequest { page: 0, kind: pdfcraft_render::RequestKind::Text, ..Default::default() }).text.unwrap(); + let range = text.find("searchable").into_iter().next().unwrap(); + for glyph in &text.glyphs[range] { + assert!(glyph.direction[0] > 0.99 && glyph.direction[1].abs() < 0.01, "{glyph:?}"); + } +} diff --git a/crates/ocr/Cargo.toml b/crates/ocr/Cargo.toml index 46bb3a1e0e..d566b3b459 100644 --- a/crates/ocr/Cargo.toml +++ b/crates/ocr/Cargo.toml @@ -13,5 +13,8 @@ workspace = true [dependencies] pdfcraft-fonts = { workspace = true } ocrs = { version = "0.13.1", default-features = false, features = ["rten"] } -rten = { version = "0.26.0", default-features = false, features = ["rten_format"] } +rten = { version = "0.26.0", default-features = false, features = ["rten_format", "onnx_format"] } +rten-tensor = "0.26.0" +rten-imageproc = "0.26.0" +image = { version = "0.25", default-features = false } thiserror.workspace = true diff --git a/crates/ocr/README.md b/crates/ocr/README.md new file mode 100644 index 0000000000..1807c0efba --- /dev/null +++ b/crates/ocr/README.md @@ -0,0 +1,36 @@ +# Local scan processing + +`cargo xtask models` downloads the hash-pinned, openly licensed models in +`ATTRIBUTION.toml`. No document is uploaded and no model is downloaded at runtime. +Release packages ship these files beside the application. + +- English, German, French, Spanish and horizontal Japanese use PaddlePaddle's + PP-OCRv6 small model through the existing Rust RTen runtime. The legacy English + recognizer API remains available to existing callers. +- PP-LCNet classifies quarter-turn orientation. Preprocessing centres on ink bounds, + resizes the short side to 256, crops to 224 and uses the model's RGB normalization. + Only predictions at or above 0.90 confidence are applied. +- Deskew searches projection profiles at bounded resolution over ±12 degrees. Weak, + blank and dense-image results stay unchanged. Fine correction scales the whole page + to fit inside its original bounds; it does not crop or recompress its images. +- OCR operates on corrected pixels, maps recognized boxes back to source coordinates, + then transforms the PDF content and invisible Unicode text together. The engine + uses empty Type 3 glyphs with ToUnicode mappings, so OCR needs no Japanese font asset. + +The UI exposes independent rotation, deskew and searchable-text switches. Automation: +`ocr_recognize` and `ocr_recognize_files` accept `language`, `auto_rotate`, `deskew` +and `recognize`; `ocr_enhance` corrects scans without adding text. Rotation/deskew are +enabled by default. Disable both for exact-image output. Existing text pages are +skipped by default. Corrections and OCR form one undo step; stale worker results and +cancelled UI runs do not modify the document. + +Limits: no perspective correction/dewarping, no guarantee on photos or ambiguous text, +no vertical Japanese claim, and deskew refuses annotated pages rather than separating +annotations from their targets. Supported OCR languages are deliberately limited to +those exposed and tested, despite the model's larger vocabulary. + +Validation: `cargo test -p pdfcraft-engine --test scan_ocr` covers actual models, +languages, all quarter-turns, deskew, unchanged exact-mode pixels, save/reopen, undo +and stale results. Model tests report a skip when models are absent. The Japanese +synthetic fixture uses the optional release `CRAFT_FONTS_DIR` checkout; generated +fixtures and third-party document samples are not committed. diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index a34386c159..b235384fcf 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -1,30 +1,32 @@ //! pdfcraft-ocr — Scan & OCR ▸ Recognize text (L4). //! -//! Recognition runs the ocrs engine (MIT/Apache-2.0) with its pre-trained models (CC-BY-SA-4.0, -//! fetched by `cargo xtask models`; see ATTRIBUTION.toml). The caller renders a page to pixels; -//! [`Ocr::recognize`] finds the words in them, and [`text_layer`] turns words placed in user space -//! into page content: invisible text (rendering mode 3) over each word, so the page becomes a -//! searchable image (Acrobat's "Searchable Image (Exact)": the image is left untouched). -//! -//! The models read the Latin alphabet (English and other languages written without accents). +//! Ocrs detects text; PaddleOCR models read English, accented Latin/Japanese and +//! classify page orientation through RTen. Models are fetched by `cargo xtask models` (see +//! ATTRIBUTION.toml), never at runtime. The engine places Unicode invisible text in PDFs; +//! [`text_layer`] remains the legacy WinAnsi content helper. See README.md for limitations. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] use std::path::{Path, PathBuf}; +mod multilingual; +pub mod scan; pub use pdfcraft_fonts::helvetica_width; /// The model files, as named in ATTRIBUTION.toml. pub const DETECTION_MODEL: &str = "text-detection.rten"; pub const RECOGNITION_MODEL: &str = "text-recognition.rten"; +pub const MULTILINGUAL_MODEL: &str = "multilingual-recognition.onnx"; +pub const ORIENTATION_MODEL: &str = "document-orientation.onnx"; +pub const DICTIONARY: &str = "multilingual-dictionary.txt"; /// The recognition model's alphabet, as it was trained (ocrs-models' `DEFAULT_ALPHABET`): its /// output classes follow this order. The ocrs crate's own copy has an `E` where the training had /// `€`, which made a scanned "€250" read as "E250". pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; -/// The languages the models read (ISO 639-1); all use the Latin alphabet without accents. -pub const LANGUAGES: &[(&str, &str)] = &[("en", "English")]; +/// Supported and tested language choices (ISO 639-1). +pub const LANGUAGES: &[(&str, &str)] = &[("en", "English"), ("de", "Deutsch"), ("ja", "日本語"), ("fr", "Français"), ("es", "Español")]; #[derive(Debug, thiserror::Error)] pub enum OcrError { @@ -40,7 +42,7 @@ pub enum OcrError { EmptyImage, } -/// Where the two model files are. +/// Where the detector and English recognizer are; multilingual assets live alongside them. #[derive(Clone, Debug, PartialEq)] pub struct Models { pub detection: PathBuf, @@ -48,10 +50,13 @@ pub struct Models { } impl Models { - /// The models in `dir`, if both files are there. + /// The models in `dir`, if every required file is there. pub fn in_dir(dir: &Path) -> Option { let m = Models { detection: dir.join(DETECTION_MODEL), recognition: dir.join(RECOGNITION_MODEL) }; - (m.detection.is_file() && m.recognition.is_file()).then_some(m) + (m.detection.is_file() + && m.recognition.is_file() + && [MULTILINGUAL_MODEL, ORIENTATION_MODEL, DICTIONARY].iter().all(|f| dir.join(f).is_file())) + .then_some(m) } /// Look for the models: `$PDFCRAFT_MODELS`, then where the release packages install them @@ -106,6 +111,7 @@ impl Line { /// A loaded recogniser. Loading takes a moment; keep one and reuse it. pub struct Ocr { engine: ocrs::OcrEngine, + multilingual: multilingual::Multilingual, } impl Ocr { @@ -118,7 +124,8 @@ impl Ocr { ..Default::default() }; let engine = ocrs::OcrEngine::new(params).map_err(|e| OcrError::Load("ocr engine".into(), e.to_string()))?; - Ok(Ocr { engine }) + let dir = models.recognition.parent().ok_or(OcrError::NoModels)?; + Ok(Ocr { engine, multilingual: multilingual::Multilingual::load(dir)? }) } /// Load the models found by [`Models::find`]. @@ -132,7 +139,8 @@ impl Ocr { return Err(OcrError::EmptyImage); } // ocrs wants 1 or 3 channels. - let rgb: std::borrow::Cow<[u8]> = if pixels.len() == (width * height * 4) as usize { + let count = (width as usize).checked_mul(height as usize).and_then(|n| n.checked_mul(4)).ok_or(OcrError::EmptyImage)?; + let rgb: std::borrow::Cow<[u8]> = if pixels.len() == count { pixels.as_chunks::<4>().0.iter().flat_map(|p| [p[0], p[1], p[2]]).collect::>().into() } else { pixels.into() @@ -160,6 +168,32 @@ impl Ocr { .filter(|l| !l.words.is_empty()) .collect()) } + + pub fn recognize_language(&self, image: &image::RgbImage, language: &str) -> Result, OcrError> { + if !LANGUAGES.iter().any(|l| l.0 == language) { + return Err(OcrError::Recognize(format!("unsupported language: {language}"))); + } + use rten_imageproc::BoundingRect; + let source = ocrs::ImageSource::from_bytes(image.as_raw(), image.dimensions()).map_err(|e| OcrError::Recognize(e.to_string()))?; + let input = self.engine.prepare_input(source).map_err(|e| OcrError::Recognize(e.to_string()))?; + let found = self.engine.detect_words(&input).map_err(|e| OcrError::Recognize(e.to_string()))?; + let boxes = self.engine.find_text_lines(&input, &found); + let mut out = Vec::new(); + for line in boxes { + let rect = line.iter().map(|r| r.bounding_rect()).fold([f32::INFINITY, f32::INFINITY, 0.0f32, 0.0f32], |a, r| { + [a[0].min(r.left()), a[1].min(r.top()), a[2].max(r.right()), a[3].max(r.bottom())] + }); + let words = self.multilingual.read_line(image, rect)?; + if !words.is_empty() { + out.push(Line { words }); + } + } + Ok(out) + } + + pub fn orientation(&self, image: &image::RgbImage) -> Result { + self.multilingual.orientation(image) + } } /// A word placed on a page, in PDF user space: the bottom-left corner of its box, and the @@ -274,7 +308,7 @@ mod tests { std::fs::create_dir_all(&models_dir).unwrap(); let found = || Models::dirs_beside_exe(&exe_dir).iter().find_map(|d| Models::in_dir(d)); assert_eq!(found(), None, "{name}: no models yet"); - for file in [DETECTION_MODEL, RECOGNITION_MODEL] { + for file in [DETECTION_MODEL, RECOGNITION_MODEL, MULTILINGUAL_MODEL, ORIENTATION_MODEL, DICTIONARY] { std::fs::write(models_dir.join(file), b"model").unwrap(); } let models = found().unwrap_or_else(|| panic!("{name}: models in {} not found", models_dir.display())); @@ -313,6 +347,10 @@ mod tests { assert!(text.iter().any(|t| t.contains("HELL") && t.contains("WOR")), "{text:?}"); let word = lines.iter().flat_map(|l| &l.words).find(|w| w.text.contains("HELL")).unwrap(); assert!(word.rect[0] >= 10.0 && word.rect[0] < 60.0, "{word:?}"); + let image = image::DynamicImage::ImageRgba8(image::RgbaImage::from_raw(w, h, px).unwrap()).to_rgb8(); + let multi = ocr.recognize_language(&image, "de").unwrap(); + assert!(multi.iter().map(Line::text).collect::>().join(" ").contains("HELLO"), "{multi:?}"); + let _ = ocr.orientation(&image).unwrap(); } mod test_image { diff --git a/crates/ocr/src/multilingual.rs b/crates/ocr/src/multilingual.rs new file mode 100644 index 0000000000..c5ef69f079 --- /dev/null +++ b/crates/ocr/src/multilingual.rs @@ -0,0 +1,142 @@ +//! PaddleOCR's Apache-2.0 PP-OCRv6 recognizer and document orientation classifier. +//! Preprocessing follows the pinned inference.yml files in ATTRIBUTION.toml. +use crate::{OcrError, Word}; +use image::{RgbImage, imageops}; +use rten_tensor::{NdTensor, Tensor, prelude::*}; + +pub(crate) struct Multilingual { + recognition: rten::Model, + orientation: rten::Model, + alphabet: Vec, +} + +fn error(e: impl std::fmt::Display) -> OcrError { + OcrError::Recognize(e.to_string()) +} + +impl Multilingual { + pub fn load(dir: &std::path::Path) -> Result { + let load = |name: &str| rten::Model::load_file(dir.join(name)).map_err(|e| OcrError::Load(name.into(), e.to_string())); + let dictionary_path = dir.join(crate::DICTIONARY); + if std::fs::metadata(&dictionary_path).map_err(error)?.len() > 1_000_000 { + return Err(error("recognition dictionary is too large")); + } + let dictionary = std::fs::read_to_string(dictionary_path).map_err(error)?; + let mut alphabet = vec![String::new()]; // CTC blank + alphabet.extend(dictionary.lines().map(str::to_owned)); + alphabet.push(" ".into()); + Ok(Self { recognition: load(crate::MULTILINGUAL_MODEL)?, orientation: load(crate::ORIENTATION_MODEL)?, alphabet }) + } + + /// Clockwise correction. Low-confidence and nearly blank/solid pages stay put. + pub fn orientation(&self, image: &RgbImage) -> Result { + let ink = image.pixels().filter(|p| p.0.iter().copied().map(u32::from).sum::() < 450).count(); + if ink < 100 || !(0.002..0.95).contains(&(ink as f64 / (image.width() as f64 * image.height() as f64).max(1.0))) { + return Ok(0); + } + // Centre the preprocessing on content. A centre crop of the paper itself can contain + // only blank margins on sparse letters, especially when scanned sideways. + let bounds = image + .enumerate_pixels() + .filter(|(_, _, p)| p.0.iter().copied().map(u32::from).sum::() < 450) + .fold([image.width(), image.height(), 0, 0], |a, (x, y, _)| [a[0].min(x), a[1].min(y), a[2].max(x + 1), a[3].max(y + 1)]); + let crop = imageops::crop_imm(image, bounds[0], bounds[1], bounds[2] - bounds[0], bounds[3] - bounds[1]).to_image(); + let image = &crop; + let scale = 256.0 / image.width().min(image.height()).max(1) as f64; + let w = (image.width() as f64 * scale).round() as u32; + let h = (image.height() as f64 * scale).round() as u32; + if w > 8192 || h > 8192 { + return Ok(0); + } + let resized = imageops::resize(image, w, h, imageops::FilterType::Triangle); + let crop = imageops::crop_imm(&resized, (w - 224) / 2, (h - 224) / 2, 224, 224).to_image(); + let mut input = NdTensor::::zeros([1, 3, 224, 224]); + for (x, y, p) in crop.enumerate_pixels() { + for c in 0..3 { + input[[0, c, y as usize, x as usize]] = (p[c] as f32 / 255.0 - [0.485, 0.456, 0.406][c]) / [0.229, 0.224, 0.225][c]; + } + } + let output: Tensor = self.orientation.run_one(input.view().into(), None).map_err(error)?.try_into().map_err(error)?; + let scores: Vec = output.iter().copied().collect(); + if scores.len() != 4 || scores.iter().any(|v| !v.is_finite()) { + return Err(error("invalid orientation model output")); + } + // The exported model includes softmax; applying it again destroys confidence. + let best = scores.iter().enumerate().max_by(|a, b| a.1.total_cmp(b.1)); + Ok(match best { + Some((i, &score)) if score >= 0.90 => ((4 - i) % 4 * 90) as u16, + _ => 0, + }) + } + + pub fn read_line(&self, image: &RgbImage, rect: [f32; 4]) -> Result, OcrError> { + let [l, t, r, b] = rect; + let x = l.floor().max(0.0) as u32; + let y = t.floor().max(0.0) as u32; + let w = (r.ceil() as u32).min(image.width()).saturating_sub(x); + let h = (b.ceil() as u32).min(image.height()).saturating_sub(y); + if w == 0 || h == 0 { + return Ok(Vec::new()); + } + let crop = imageops::crop_imm(image, x, y, w, h).to_image(); + let rw = ((48.0 * w as f64 / h as f64).ceil() as u32).clamp(1, 3200); + let width = rw.max(320) as usize; + let resized = imageops::resize(&crop, rw, 48, imageops::FilterType::Triangle); + let mut input = NdTensor::::zeros([1, 3, 48, width]); + // Paddle's recognizer takes BGR in [-1,1], with zero right-padding. + for (x, y, p) in resized.enumerate_pixels() { + for c in 0..3 { + input[[0, c, y as usize, x as usize]] = p[2 - c] as f32 / 127.5 - 1.0; + } + } + let output: Tensor = self.recognition.run_one(input.view().into(), None).map_err(error)?.try_into().map_err(error)?; + let shape = output.shape(); + if shape.len() != 3 || shape[0] != 1 || shape[2] != self.alphabet.len() { + return Err(error(format!("invalid recognition model output: {shape:?}, dictionary {}", self.alphabet.len()))); + } + let steps = shape[1]; + let classes = shape[2]; + let values: Vec = output.iter().copied().collect(); + let mut words = Vec::new(); + let (mut text, mut previous, mut start, mut end, mut confidence, mut count) = (String::new(), 0, 0, 0, 0.0, 0); + let scale = width as f32 / rw as f32 * w as f32 / steps.max(1) as f32; + let flush = |words: &mut Vec, text: &mut String, start: usize, end: usize, confidence: f32, count: usize| { + if !text.is_empty() && confidence / count.max(1) as f32 >= 0.5 { + words.push(Word { + text: std::mem::take(text), + rect: [x as f32 + start as f32 * scale, y as f32, (x as f32 + (end + 1) as f32 * scale).min(r), (y + h) as f32], + }); + } else { + text.clear(); + } + }; + for (step, row) in values.chunks_exact(classes).enumerate() { + if step as f32 * scale >= w as f32 { + break; + } + let Some((index, &score)) = row.iter().enumerate().max_by(|a, b| a.1.total_cmp(b.1)) else { continue }; + if index != 0 && index != previous { + let Some(ch) = self.alphabet.get(index) else { continue }; + if ch.trim().is_empty() { + flush(&mut words, &mut text, start, end, confidence, count); + confidence = 0.0; + count = 0; + } else { + if text.is_empty() { + start = step; + } + text.push_str(ch); + end = step; + confidence += score; + count += 1; + } + } + if index != 0 && index == previous && !text.is_empty() { + end = step; + } + previous = index; + } + flush(&mut words, &mut text, start, end, confidence, count); + Ok(words) + } +} diff --git a/crates/ocr/src/scan.rs b/crates/ocr/src/scan.rs new file mode 100644 index 0000000000..b455a61806 --- /dev/null +++ b/crates/ocr/src/scan.rs @@ -0,0 +1,149 @@ +//! Bounded scan geometry. Corrections fit inside the page, never crop away an edge. +use image::{Rgb, RgbImage, imageops}; + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Correction { + /// Clockwise quarter-turn, applied first. + pub rotation: u16, + /// Clockwise fine rotation, in degrees (-12..12). + pub deskew: f64, +} + +impl Correction { + pub fn changed(self) -> bool { + self.rotation != 0 || self.deskew.abs() >= 0.05 + } + pub fn scale(self, width: f64, height: f64) -> f64 { + let (s, c) = self.deskew.to_radians().sin_cos(); + (width / (width * c.abs() + height * s.abs())).min(height / (width * s.abs() + height * c.abs())) + } + /// Corrected pixel coordinates back to the original rendered page (edge coordinates). + pub fn to_source(self, x: f64, y: f64, width: f64, height: f64) -> [f64; 2] { + let (w, h) = if self.rotation % 180 == 90 { (height, width) } else { (width, height) }; + let k = self.scale(w, h); + let (s, c) = self.deskew.to_radians().sin_cos(); + let (dx, dy) = ((x - w / 2.0) / k, (y - h / 2.0) / k); + let (x, y) = (c * dx + s * dy + w / 2.0, -s * dx + c * dy + h / 2.0); + match self.rotation { + 90 => [y, height - x], + 180 => [width - x, height - y], + 270 => [width - y, x], + _ => [x, y], + } + } + pub fn apply(self, image: &RgbImage) -> RgbImage { + let turned = match self.rotation { + 90 => imageops::rotate90(image), + 180 => imageops::rotate180(image), + 270 => imageops::rotate270(image), + _ => image.clone(), + }; + if self.deskew.abs() < 0.05 { + return turned; + } + let (w, h) = turned.dimensions(); + let inverse = Self { rotation: 0, ..self }; + RgbImage::from_fn(w, h, |x, y| { + let [sx, sy] = inverse.to_source(x as f64 + 0.5, y as f64 + 0.5, w as f64, h as f64); + let (sx, sy) = (sx - 0.5, sy - 0.5); + let (x0, y0) = (sx.floor() as i64, sy.floor() as i64); + let (fx, fy) = (sx - sx.floor(), sy - sy.floor()); + let sample = |dx: i64, dy: i64| { + if x0 + dx >= 0 && y0 + dy >= 0 { + turned.get_pixel_checked((x0 + dx) as u32, (y0 + dy) as u32).copied().unwrap_or(Rgb([255; 3])) + } else { + Rgb([255; 3]) + } + }; + let p = [sample(0, 0), sample(1, 0), sample(0, 1), sample(1, 1)]; + Rgb(std::array::from_fn(|c| { + ((p[0][c] as f64 * (1.0 - fx) + p[1][c] as f64 * fx) * (1.0 - fy) + (p[2][c] as f64 * (1.0 - fx) + p[3][c] as f64 * fx) * fy).round() + as u8 + })) + }) + } +} + +/// Projection-profile deskew, at bounded resolution. Blank/photo-like pages and weak peaks +/// are left alone. Search both directions and refine the strongest horizontal text baseline. +pub fn deskew(image: &RgbImage) -> f64 { + let ratio = 1000.0 / image.width().max(image.height()).max(1) as f64; + let small = if ratio < 1.0 { + imageops::resize( + image, + (image.width() as f64 * ratio).round().max(1.0) as u32, + (image.height() as f64 * ratio).round().max(1.0) as u32, + imageops::FilterType::Triangle, + ) + } else { + image.clone() + }; + let (w, h) = small.dimensions(); + let points: Vec<(f64, f64)> = small + .enumerate_pixels() + .filter(|(_, _, p)| p.0.iter().copied().map(u32::from).sum::() < 420) + .map(|(x, y, _)| (x as f64 - w as f64 / 2.0, y as f64 - h as f64 / 2.0)) + .collect(); + let fraction = points.len() as f64 / (w as f64 * h as f64).max(1.0); + if points.len() < 100 || !(0.002..0.35).contains(&fraction) { + return 0.0; + } + let stride = points.len().div_ceil(100_000).max(1); + let bins = (w + h + 4) as usize; + let score = |degrees: f64| { + let (s, c) = degrees.to_radians().sin_cos(); + let mut rows = vec![0u32; bins]; + for &(x, y) in points.iter().step_by(stride) { + let bin = (s * x + c * y + bins as f64 / 2.0).round() as usize; + if let Some(n) = rows.get_mut(bin) { + *n += 1; + } + } + rows.iter().map(|&n| (n as f64).powi(2)).sum::() + }; + let baseline = score(0.0); + let (mut best, mut value) = (0.0, baseline); + for n in -48..=48 { + let a = n as f64 / 4.0; + let v = score(a); + if v > value { + best = a; + value = v; + } + } + if best.abs() >= 12.0 || value < baseline * 1.15 { + return 0.0; + } + let coarse = best; + for n in -5..=5 { + let a = coarse + n as f64 * 0.05; + let v = score(a); + if v > value { + best = a; + value = v; + } + } + if best.abs() < 0.15 { 0.0 } else { best } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn blank_and_solid_images_are_not_deskewed() { + for color in [0, 255] { + assert_eq!(deskew(&RgbImage::from_pixel(600, 800, Rgb([color; 3]))), 0.0); + } + } + #[test] + fn deskew_finds_both_signs_and_keeps_corners() { + let image = RgbImage::from_fn(600, 800, |x, y| { + if (80..520).contains(&x) && (80..720).contains(&y) && y % 40 < 4 && x % 30 < 22 { Rgb([0; 3]) } else { Rgb([255; 3]) } + }); + for angle in [-5.0, 3.0] { + let tilted = Correction { rotation: 0, deskew: angle }.apply(&image); + assert!((deskew(&tilted) + angle).abs() < 0.2, "{}", deskew(&tilted)); + } + assert_eq!(Correction { rotation: 90, deskew: 0.0 }.to_source(0.0, 0.0, 600.0, 800.0), [0.0, 800.0]); + } +} diff --git a/crates/ui-egui/src/commands.rs b/crates/ui-egui/src/commands.rs index 8ccf145626..c3e49673cb 100644 --- a/crates/ui-egui/src/commands.rs +++ b/crates/ui-egui/src/commands.rs @@ -353,7 +353,14 @@ impl PdfCraftApp { "export.image" => self.dialog = Some(Dialog::Export(crate::export_ui::ExportKind::Image)), "export.text" => self.dialog = Some(Dialog::Export(crate::export_ui::ExportKind::Text)), "a11y.check" => self.start_accessibility_check(), - "ocr.recognize" => self.dialog = Some(Dialog::RecognizeText), + "ocr.recognize" => { + self.ocr_draft.recognize = true; + self.dialog = Some(Dialog::RecognizeText); + } + "ocr.enhance" => { + self.ocr_draft.recognize = false; + self.dialog = Some(Dialog::RecognizeText); + } "tools.js_console" => self.dialog = Some(Dialog::JsConsole), "tools.document_js" => { self.doc_js = Default::default(); diff --git a/crates/ui-egui/src/i18n/ar.tsv b/crates/ui-egui/src/i18n/ar.tsv index 48d5baf08c..1c3654ace9 100644 --- a/crates/ui-egui/src/i18n/ar.tsv +++ b/crates/ui-egui/src/i18n/ar.tsv @@ -2214,3 +2214,16 @@ key Space مسافة Panels اللوحات Resize panels تغيير حجم اللوحات Resize panel window تغيير حجم نافذة اللوحة + Scan and text recognition المسح والتعرف على النص + Searchable Image صورة قابلة للبحث + Corrected scan مسح مصحح + Scan correction تصحيح المسح + Automatically rotate sideways and upside-down pages تدوير الصفحات الجانبية والمقلوبة تلقائيًا + Straighten skewed scans تقويم الصفحات الممسوحة المائلة + Add searchable text إضافة نص قابل للبحث + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. تبقى الاتجاهات غير المؤكدة دون تغيير. يحافظ التقويم على الصورة كاملة داخل الصفحة. + Correct scans تصحيح الصفحات الممسوحة + Scan processing cancelled أُلغيت معالجة المسح + ; 1 page skipped ; تم تخطي صفحة واحدة + ; {n} pages skipped ; تم تخطي {n} صفحات + ; corrected {n} pages ; تم تصحيح {n} صفحات diff --git a/crates/ui-egui/src/i18n/bg.tsv b/crates/ui-egui/src/i18n/bg.tsv index 5c514ac7c8..6ac063db2e 100644 --- a/crates/ui-egui/src/i18n/bg.tsv +++ b/crates/ui-egui/src/i18n/bg.tsv @@ -2156,3 +2156,16 @@ key Space Интервал Resize panel window Преоразмери прозореца на панела Tools Инструменти Inspector Инспектор + Scan and text recognition Сканиране и разпознаване на текст + Searchable Image Изображение с търсене + Corrected scan Коригирано сканиране + Scan correction Корекция на сканиране + Automatically rotate sideways and upside-down pages Автоматично завъртане на странични и обърнати страници + Straighten skewed scans Изправяне на наклонени сканирани страници + Add searchable text Добавяне на текст за търсене + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Несигурната ориентация остава непроменена. Изправянето побира цялото изображение в страницата. + Correct scans Коригиране на сканирания + Scan processing cancelled Обработката на сканиранията е отменена + ; 1 page skipped ; пропусната 1 страница + ; {n} pages skipped ; пропуснати страници: {n} + ; corrected {n} pages ; коригирани страници: {n} diff --git a/crates/ui-egui/src/i18n/cs.tsv b/crates/ui-egui/src/i18n/cs.tsv index 0d4741fabb..2fb2a7c5a4 100644 --- a/crates/ui-egui/src/i18n/cs.tsv +++ b/crates/ui-egui/src/i18n/cs.tsv @@ -182,3 +182,16 @@ key Space Mezerník Resize panel window Změnit velikost okna panelu Tools Nástroje Inspector Inspektor + Scan and text recognition Skenování a rozpoznávání textu + Searchable Image Prohledávatelný obrázek + Corrected scan Opravený sken + Scan correction Oprava skenu + Automatically rotate sideways and upside-down pages Automaticky otáčet stránky na boku a vzhůru nohama + Straighten skewed scans Narovnat šikmé skeny + Add searchable text Přidat prohledávatelný text + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Nejistá orientace zůstane beze změny. Narovnání zachová celý obrázek uvnitř stránky. + Correct scans Opravit skeny + Scan processing cancelled Zpracování skenů zrušeno + ; 1 page skipped ; přeskočena 1 stránka + ; {n} pages skipped ; přeskočeno stránek: {n} + ; corrected {n} pages ; opraveno stránek: {n} diff --git a/crates/ui-egui/src/i18n/de.tsv b/crates/ui-egui/src/i18n/de.tsv index ee07c925da..b5fd9cafdf 100644 --- a/crates/ui-egui/src/i18n/de.tsv +++ b/crates/ui-egui/src/i18n/de.tsv @@ -2194,3 +2194,16 @@ key Space Leertaste Resize panel window Größe des Bereichsfensters ändern Tools Werkzeuge Inspector Inspektor + Scan and text recognition Scan und Texterkennung + Searchable Image Durchsuchbares Bild + Corrected scan Korrigierter Scan + Scan correction Scan korrigieren + Automatically rotate sideways and upside-down pages Seitlich oder kopfstehend gescannte Seiten automatisch drehen + Straighten skewed scans Schief eingescannte Seiten geraderichten + Add searchable text Durchsuchbaren Text hinzufügen + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Unsichere Ausrichtungen bleiben unverändert. Beim Geraderichten bleibt das gesamte Bild innerhalb der Seite. + Correct scans Scans korrigieren + Scan processing cancelled Scan-Verarbeitung abgebrochen + ; 1 page skipped ; 1 Seite übersprungen + ; {n} pages skipped ; {n} Seiten übersprungen + ; corrected {n} pages ; {n} Seiten korrigiert diff --git a/crates/ui-egui/src/i18n/es.tsv b/crates/ui-egui/src/i18n/es.tsv index c7cced68b9..64af66d417 100644 --- a/crates/ui-egui/src/i18n/es.tsv +++ b/crates/ui-egui/src/i18n/es.tsv @@ -2110,3 +2110,16 @@ key Space Espacio Resize panel window Cambiar tamaño de la ventana del panel Tools Herramientas Inspector Inspector + Scan and text recognition Escaneo y reconocimiento de texto + Searchable Image Imagen con texto buscable + Corrected scan Escaneo corregido + Scan correction Corrección de escaneo + Automatically rotate sideways and upside-down pages Girar automáticamente las páginas de lado o boca abajo + Straighten skewed scans Enderezar las páginas escaneadas torcidas + Add searchable text Añadir texto buscable + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Las orientaciones dudosas no se modifican. Al enderezar, toda la imagen cabe en la página. + Correct scans Corregir escaneos + Scan processing cancelled Procesamiento de escaneos cancelado + ; 1 page skipped ; 1 página omitida + ; {n} pages skipped ; {n} páginas omitidas + ; corrected {n} pages ; {n} páginas corregidas diff --git a/crates/ui-egui/src/i18n/fr.tsv b/crates/ui-egui/src/i18n/fr.tsv index 79cda906ed..19be0e4935 100644 --- a/crates/ui-egui/src/i18n/fr.tsv +++ b/crates/ui-egui/src/i18n/fr.tsv @@ -2194,3 +2194,16 @@ key Space Espace Resize panel window Redimensionner la fenêtre du panneau Tools Outils Inspector Inspecteur + Scan and text recognition Numérisation et reconnaissance de texte + Searchable Image Image consultable + Corrected scan Numérisation corrigée + Scan correction Correction de numérisation + Automatically rotate sideways and upside-down pages Pivoter automatiquement les pages de côté ou à l’envers + Straighten skewed scans Redresser les pages numérisées de travers + Add searchable text Ajouter du texte consultable + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Les orientations incertaines restent inchangées. Le redressement conserve toute l’image dans la page. + Correct scans Corriger les numérisations + Scan processing cancelled Traitement des numérisations annulé + ; 1 page skipped ; 1 page ignorée + ; {n} pages skipped ; {n} pages ignorées + ; corrected {n} pages ; {n} pages corrigées diff --git a/crates/ui-egui/src/i18n/hu.tsv b/crates/ui-egui/src/i18n/hu.tsv index 7fb60fb4b0..9636844976 100644 --- a/crates/ui-egui/src/i18n/hu.tsv +++ b/crates/ui-egui/src/i18n/hu.tsv @@ -2192,3 +2192,16 @@ key Space Szóköz Resize panel window Panelablak átméretezése Tools Eszközök Inspector Tulajdonságvizsgáló + Scan and text recognition Szkennelés és szövegfelismerés + Searchable Image Kereshető kép + Corrected scan Javított szkennelés + Scan correction Szkennelés javítása + Automatically rotate sideways and upside-down pages Oldalra vagy fejjel lefelé álló oldalak automatikus elforgatása + Straighten skewed scans Ferde szkennelt oldalak kiegyenesítése + Add searchable text Kereshető szöveg hozzáadása + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. A bizonytalan tájolás változatlan marad. A kiegyenesítés a teljes képet az oldalon belül tartja. + Correct scans Szkennelések javítása + Scan processing cancelled Szkennelések feldolgozása megszakítva + ; 1 page skipped ; 1 oldal kihagyva + ; {n} pages skipped ; {n} oldal kihagyva + ; corrected {n} pages ; {n} oldal javítva diff --git a/crates/ui-egui/src/i18n/it.tsv b/crates/ui-egui/src/i18n/it.tsv index 01ea9d636b..eb5effa7e3 100644 --- a/crates/ui-egui/src/i18n/it.tsv +++ b/crates/ui-egui/src/i18n/it.tsv @@ -2201,3 +2201,16 @@ key Space Spazio Panels Pannelli Resize panels Ridimensiona pannelli Resize panel window Ridimensiona finestra pannello + Scan and text recognition Scansione e riconoscimento del testo + Searchable Image Immagine ricercabile + Corrected scan Scansione corretta + Scan correction Correzione scansione + Automatically rotate sideways and upside-down pages Ruota automaticamente le pagine laterali o capovolte + Straighten skewed scans Raddrizza le scansioni inclinate + Add searchable text Aggiungi testo ricercabile + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Gli orientamenti incerti restano invariati. Il raddrizzamento mantiene l’intera immagine nella pagina. + Correct scans Correggi scansioni + Scan processing cancelled Elaborazione scansioni annullata + ; 1 page skipped ; 1 pagina saltata + ; {n} pages skipped ; {n} pagine saltate + ; corrected {n} pages ; {n} pagine corrette diff --git a/crates/ui-egui/src/i18n/ja.tsv b/crates/ui-egui/src/i18n/ja.tsv index f782fa4426..fff3a7322e 100644 --- a/crates/ui-egui/src/i18n/ja.tsv +++ b/crates/ui-egui/src/i18n/ja.tsv @@ -2194,3 +2194,16 @@ key Space スペース Resize panel window パネルウィンドウのサイズを変更 Tools ツール Inspector インスペクター + Scan and text recognition スキャンと文字認識 + Searchable Image 検索可能な画像 + Corrected scan 補正済みスキャン + Scan correction スキャン補正 + Automatically rotate sideways and upside-down pages 横向きや上下逆のページを自動回転 + Straighten skewed scans スキャンの傾きを補正 + Add searchable text 検索可能なテキストを追加 + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. 向きが不確かな場合は変更しません。傾き補正では画像全体をページ内に収めます。 + Correct scans スキャンを補正 + Scan processing cancelled スキャン処理をキャンセルしました + ; 1 page skipped ; 1 ページをスキップ + ; {n} pages skipped ; {n} ページをスキップ + ; corrected {n} pages ; {n} ページを補正 diff --git a/crates/ui-egui/src/i18n/pl.tsv b/crates/ui-egui/src/i18n/pl.tsv index 64485e69e8..729d01c895 100644 --- a/crates/ui-egui/src/i18n/pl.tsv +++ b/crates/ui-egui/src/i18n/pl.tsv @@ -2189,3 +2189,16 @@ key Space Spacja Text used as a clipping path cannot be moved with this selection. Tekstu używanego jako ścieżka przycinania nie można przenieść razem z tym zaznaczeniem. These objects could not be moved. Select them again and try a smaller move. Nie udało się przenieść tych obiektów. Zaznacz je ponownie i spróbuj mniejszego przesunięcia. This page contains objects that cannot be selected together. Ta strona zawiera obiekty, których nie można zaznaczyć razem. + Scan and text recognition Skanowanie i rozpoznawanie tekstu + Searchable Image Obraz z możliwością wyszukiwania + Corrected scan Poprawiony skan + Scan correction Korekcja skanu + Automatically rotate sideways and upside-down pages Automatycznie obracaj strony boczne i odwrócone + Straighten skewed scans Prostuj przekrzywione skany + Add searchable text Dodaj tekst do wyszukiwania + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Niepewna orientacja pozostaje bez zmian. Prostowanie mieści cały obraz na stronie. + Correct scans Popraw skany + Scan processing cancelled Przetwarzanie skanów anulowane + ; 1 page skipped ; pominięto 1 stronę + ; {n} pages skipped ; pominięto stron: {n} + ; corrected {n} pages ; poprawiono stron: {n} diff --git a/crates/ui-egui/src/i18n/pt-br.tsv b/crates/ui-egui/src/i18n/pt-br.tsv index 697f7585f1..94249d56c8 100644 --- a/crates/ui-egui/src/i18n/pt-br.tsv +++ b/crates/ui-egui/src/i18n/pt-br.tsv @@ -2195,3 +2195,16 @@ key Space Espaço Resize panel window Redimensionar janela do painel Tools Ferramentas Inspector Inspetor + Scan and text recognition Digitalização e reconhecimento de texto + Searchable Image Imagem pesquisável + Corrected scan Digitalização corrigida + Scan correction Correção de digitalização + Automatically rotate sideways and upside-down pages Girar automaticamente páginas de lado ou de cabeça para baixo + Straighten skewed scans Endireitar digitalizações inclinadas + Add searchable text Adicionar texto pesquisável + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Orientações incertas permanecem inalteradas. Ao endireitar, toda a imagem cabe na página. + Correct scans Corrigir digitalizações + Scan processing cancelled Processamento das digitalizações cancelado + ; 1 page skipped ; 1 página ignorada + ; {n} pages skipped ; {n} páginas ignoradas + ; corrected {n} pages ; {n} páginas corrigidas diff --git a/crates/ui-egui/src/i18n/ru.tsv b/crates/ui-egui/src/i18n/ru.tsv index 711cd09563..cf6548bd74 100644 --- a/crates/ui-egui/src/i18n/ru.tsv +++ b/crates/ui-egui/src/i18n/ru.tsv @@ -2107,3 +2107,16 @@ key Space Пробел Resize panel window Изменить размер окна панели Tools Инструменты Inspector Инспектор + Scan and text recognition Сканирование и распознавание текста + Searchable Image Изображение с поиском + Corrected scan Исправленный скан + Scan correction Исправление скана + Automatically rotate sideways and upside-down pages Автоматически поворачивать боковые и перевёрнутые страницы + Straighten skewed scans Выравнивать наклонённые сканы + Add searchable text Добавить текст для поиска + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Неуверенно определённая ориентация не меняется. При выравнивании всё изображение помещается на странице. + Correct scans Исправить сканы + Scan processing cancelled Обработка сканов отменена + ; 1 page skipped ; пропущена 1 страница + ; {n} pages skipped ; пропущено страниц: {n} + ; corrected {n} pages ; исправлено страниц: {n} diff --git a/crates/ui-egui/src/i18n/te.tsv b/crates/ui-egui/src/i18n/te.tsv index f3a4afd332..2d6edd0e76 100644 --- a/crates/ui-egui/src/i18n/te.tsv +++ b/crates/ui-egui/src/i18n/te.tsv @@ -2124,3 +2124,16 @@ key Space స్పేస్ Resize panel window ప్యానెల్ విండో పరిమాణం మార్చు Tools సాధనాలు Inspector పరిశీలకుడు + Scan and text recognition స్కాన్ మరియు అక్షర గుర్తింపు + Searchable Image శోధించగల చిత్రం + Corrected scan సరిచేసిన స్కాన్ + Scan correction స్కాన్ సవరణ + Automatically rotate sideways and upside-down pages పక్కకు లేదా తలక్రిందులుగా ఉన్న పేజీలను స్వయంచాలకంగా తిప్పు + Straighten skewed scans వంగిన స్కాన్‌లను నిటారుగా చేయి + Add searchable text శోధించగల పాఠ్యాన్ని చేర్చు + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. దిశ స్పష్టంగా లేనప్పుడు మార్చబడదు. నిటారుగా చేసేటప్పుడు మొత్తం చిత్రం పేజీలోనే ఉంటుంది. + Correct scans స్కాన్‌లను సరిచేయి + Scan processing cancelled స్కాన్ ప్రాసెసింగ్ రద్దయింది + ; 1 page skipped ; 1 పేజీ దాటవేయబడింది + ; {n} pages skipped ; {n} పేజీలు దాటవేయబడ్డాయి + ; corrected {n} pages ; {n} పేజీలు సరిచేయబడ్డాయి diff --git a/crates/ui-egui/src/i18n/uk.tsv b/crates/ui-egui/src/i18n/uk.tsv index b1f565c8c9..4035c5f086 100644 --- a/crates/ui-egui/src/i18n/uk.tsv +++ b/crates/ui-egui/src/i18n/uk.tsv @@ -2215,3 +2215,16 @@ key Space Пробіл Panels Панелі Resize panels Змінити розмір панелей Resize panel window Змінити розмір вікна панелі + Scan and text recognition Сканування та розпізнавання тексту + Searchable Image Зображення з пошуком + Corrected scan Виправлений скан + Scan correction Виправлення скану + Automatically rotate sideways and upside-down pages Автоматично повертати бокові та перевернуті сторінки + Straighten skewed scans Вирівнювати нахилені скани + Add searchable text Додати текст для пошуку + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. Непевно визначена орієнтація не змінюється. Під час вирівнювання все зображення вміщується на сторінці. + Correct scans Виправити скани + Scan processing cancelled Обробку сканів скасовано + ; 1 page skipped ; пропущено 1 сторінку + ; {n} pages skipped ; пропущено сторінок: {n} + ; corrected {n} pages ; виправлено сторінок: {n} diff --git a/crates/ui-egui/src/i18n/zh-hans.tsv b/crates/ui-egui/src/i18n/zh-hans.tsv index 898a66bcc1..27dfbe1c10 100644 --- a/crates/ui-egui/src/i18n/zh-hans.tsv +++ b/crates/ui-egui/src/i18n/zh-hans.tsv @@ -2158,3 +2158,16 @@ key Space 空格 Resize panel window 调整面板窗口大小 Tools 工具 Inspector 检查器 + Scan and text recognition 扫描和文字识别 + Searchable Image 可搜索的图像 + Corrected scan 已校正的扫描件 + Scan correction 扫描校正 + Automatically rotate sideways and upside-down pages 自动旋转侧向和倒置的页面 + Straighten skewed scans 校正扫描件倾斜 + Add searchable text 添加可搜索文本 + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. 方向不确定时保持不变。倾斜校正会将整个图像保留在页面内。 + Correct scans 校正扫描件 + Scan processing cancelled 扫描处理已取消 + ; 1 page skipped ; 已跳过 1 页 + ; {n} pages skipped ; 已跳过 {n} 页 + ; corrected {n} pages ; 已校正 {n} 页 diff --git a/crates/ui-egui/src/i18n/zh-hant.tsv b/crates/ui-egui/src/i18n/zh-hant.tsv index 2f1936d331..dcf5d6941d 100644 --- a/crates/ui-egui/src/i18n/zh-hant.tsv +++ b/crates/ui-egui/src/i18n/zh-hant.tsv @@ -2080,3 +2080,16 @@ key Space 空格 Resize panel window 調整面板視窗大小 Tools 工具 Inspector 檢查器 + Scan and text recognition 掃描與文字辨識 + Searchable Image 可搜尋的影像 + Corrected scan 已校正的掃描件 + Scan correction 掃描校正 + Automatically rotate sideways and upside-down pages 自動旋轉側向和倒置的頁面 + Straighten skewed scans 校正掃描件傾斜 + Add searchable text 新增可搜尋文字 + Uncertain orientations are left unchanged. Deskew fits the full image inside the page. 方向不確定時保持不變。傾斜校正會將整個影像保留在頁面內。 + Correct scans 校正掃描件 + Scan processing cancelled 掃描處理已取消 + ; 1 page skipped ; 已略過 1 頁 + ; {n} pages skipped ; 已略過 {n} 頁 + ; corrected {n} pages ; 已校正 {n} 頁 diff --git a/crates/ui-egui/src/ocr_ui.rs b/crates/ui-egui/src/ocr_ui.rs index f980d7cc02..077f604327 100644 --- a/crates/ui-egui/src/ocr_ui.rs +++ b/crates/ui-egui/src/ocr_ui.rs @@ -28,11 +28,14 @@ pub struct OcrDraft { pub language: String, /// "Downsample to" resolution. pub dpi: u32, + pub auto_rotate: bool, + pub deskew: bool, + pub recognize: bool, } impl Default for OcrDraft { fn default() -> Self { - OcrDraft { pages: OcrPages::All, from: 1, to: 1, language: "en".into(), dpi: 300 } + OcrDraft { pages: OcrPages::All, from: 1, to: 1, language: "en".into(), dpi: 300, auto_rotate: true, deskew: true, recognize: true } } } @@ -64,7 +67,7 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo let d = &mut app.ocr_draft; d.to = d.to.clamp(1, pages); d.from = d.from.clamp(1, d.to); - ui.label(egui::RichText::new(tl!("Recognize Text")).font(theme::semibold(18.0))); + ui.label(egui::RichText::new(tl!("Scan and text recognition")).font(theme::semibold(18.0))); ui.add_space(8.0); let group = |ui: &mut egui::Ui, title: &str, body: &mut dyn FnMut(&mut egui::Ui)| { ui.label(egui::RichText::new(tl!(title)).font(theme::semibold(13.0))); @@ -96,10 +99,12 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo }); ui.end_row(); ui.label(tl!("Output")); - egui::ComboBox::from_id_salt("ocr-output").selected_text(tl!("Searchable Image (Exact)")).width(220.0).show_ui(ui, |ui| { - let _ = ui - .selectable_label(true, tl!("Searchable Image (Exact)")) - .on_hover_text(tl!("Adds invisible text over each word; the page image is not changed")); + ui.label(if !d.recognize { + tl!("Corrected scan") + } else if d.auto_rotate || d.deskew { + tl!("Searchable Image") + } else { + tl!("Searchable Image (Exact)") }); ui.end_row(); ui.label(tl!("Downsample to")); @@ -111,6 +116,16 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo ui.end_row(); }); }); + group(ui, "Scan correction", &mut |ui| { + ui.checkbox(&mut d.auto_rotate, tl!("Automatically rotate sideways and upside-down pages")); + ui.checkbox(&mut d.deskew, tl!("Straighten skewed scans")); + ui.checkbox(&mut d.recognize, tl!("Add searchable text")); + ui.label( + egui::RichText::new(tl!("Uncertain orientations are left unchanged. Deskew fits the full image inside the page.")) + .small() + .color(t.text_muted), + ); + }); if !available { ui.label( egui::RichText::new(tl!("Text recognition isn't installed: its model files are missing. Reinstall PdfCraft, or set PDFCRAFT_MODELS to the folder that holds them.")) @@ -123,7 +138,13 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo let (mut go, mut cancel) = (false, false); ui.horizontal(|ui| { ui.with_layout(Layout::right_to_left(Align::Center), |ui| { - if ui.add_enabled_ui(available, |ui| widgets::pill_button(ui, tl!("Recognize text"), true)).inner.clicked() { + if ui + .add_enabled_ui(available, |ui| { + widgets::pill_button(ui, if d.recognize { tl!("Recognize text") } else { tl!("Correct scans") }, true) + }) + .inner + .clicked() + { go = true; } if widgets::pill_button(ui, tl!("Cancel"), false).clicked() { @@ -152,7 +173,14 @@ impl PdfCraftApp { OcrPages::Current => vec![self.views[vi].current], OcrPages::Range => (d.from.saturating_sub(1)..d.to).collect(), }; - let settings = OcrSettings { dpi: d.dpi as f32, language: d.language.clone(), ..Default::default() }; + let settings = OcrSettings { + dpi: d.dpi as f32, + language: d.language.clone(), + auto_rotate: d.auto_rotate, + deskew: d.deskew, + recognize: d.recognize, + ..Default::default() + }; let Some(job) = self.session.ocr_job(id, &pages, settings) else { return }; let progress = Arc::new(Mutex::new(OcrProgress { total: job.pages.len(), ..Default::default() })); let p = progress.clone(); @@ -189,7 +217,15 @@ impl PdfCraftApp { return; } // The settings showing now, not when the folder arrives. - let settings = OcrSettings { dpi: self.ocr_draft.dpi as f32, language: self.ocr_draft.language.clone(), ..Default::default() }; + let d = &self.ocr_draft; + let settings = OcrSettings { + dpi: d.dpi as f32, + language: d.language.clone(), + auto_rotate: d.auto_rotate, + deskew: d.deskew, + recognize: d.recognize, + ..Default::default() + }; #[cfg(not(target_arch = "wasm32"))] match self.export_dir_override.clone() { Some(d) => self.ocr_files_into(files, settings, d.into()), @@ -267,7 +303,7 @@ impl PdfCraftApp { self.poll_ocr(); } - /// Stop a running recognition (the pages read so far are kept). + /// Stop a running recognition without applying partial results. pub fn cancel_ocr(&mut self) { if let Some(r) = &self.ocr_run && let Ok(mut s) = r.progress.lock() @@ -317,8 +353,13 @@ impl PdfCraftApp { } return; }; + let cancelled = s.cancel; drop(s); self.ocr_run = None; + if cancelled { + self.notify_tr("Scan processing cancelled"); + return; + } let found = match result { Ok(f) => f, Err(e) => { @@ -328,6 +369,7 @@ impl PdfCraftApp { }; let read = found.iter().filter(|p| p.skipped.is_none()).count(); let skipped = found.len() - read; + let corrected = found.iter().filter(|p| p.skipped.is_none() && p.correction.changed()).count(); match self.session.apply_ocr(doc, &found) { Ok(words) => { if let Some(info) = self.session.get(doc).map(|d| d.info.clone()) @@ -344,11 +386,19 @@ impl PdfCraftApp { let mut msg = crate::i18n::fmt(tl!("{words} on {pages}"), &[("words", &words_part), ("pages", &pages_part)]); if skipped > 0 { msg.push_str(&if skipped == 1 { - crate::i18n::fmt(tl!("; 1 page already had text"), &[]) + crate::i18n::fmt(tl!("; 1 page skipped"), &[]) } else { - crate::i18n::fmt(tl!("; {n} pages already had text"), &[("n", &skipped.to_string())]) + crate::i18n::fmt(tl!("; {n} pages skipped"), &[("n", &skipped.to_string())]) }); } + if corrected > 0 { + msg.push_str(&crate::i18n::fmt(tl!("; corrected {n} pages"), &[("n", &corrected.to_string())])); + } + let reasons: std::collections::BTreeSet<_> = found.iter().filter_map(|p| p.skipped.as_deref()).collect(); + if !reasons.is_empty() { + msg.push_str(": "); + msg.push_str(&reasons.into_iter().collect::>().join("; ")); + } self.notify(msg); } Err(e) => self.notify_error(e), diff --git a/crates/ui-egui/tests/ocr.rs b/crates/ui-egui/tests/ocr.rs index a62661fd1e..30cfcf0895 100644 --- a/crates/ui-egui/tests/ocr.rs +++ b/crates/ui-egui/tests/ocr.rs @@ -29,6 +29,9 @@ fn recognize_text_dialog_adds_searchable_text() { h.run_steps(2); h.get_by_label("Document language"); h.get_by_label("Output"); + h.get_by_label("Automatically rotate sideways and upside-down pages"); + h.get_by_label("Straighten skewed scans"); + assert!(h.state().ocr_draft.auto_rotate && h.state().ocr_draft.deskew); if !pdfcraft_engine::ocr::available() { eprintln!("skipped: OCR models not installed"); return; @@ -43,6 +46,24 @@ fn recognize_text_dialog_adds_searchable_text() { assert_eq!(app.session.get(id).unwrap().can_undo(), Some("Recognize text")); } +#[test] +fn enhance_scan_dialog_offers_correction_without_recognition() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.enhance")); + h.run_steps(2); + h.get_by_label("Correct scans"); + assert!(!h.state().ocr_draft.recognize); + h.get_by_label("Add searchable text").click(); + h.run_steps(2); + assert!(h.state().ocr_draft.recognize); +} + #[test] fn recognize_text_in_multiple_files_writes_searchable_copies() { if !pdfcraft_engine::ocr::available() { diff --git a/docs/gaps.md b/docs/gaps.md index 262e4c7d7d..315e3e21bf 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -134,18 +134,16 @@ update the numbers. (`AGENTS.md` §4). - **Estimate:** 45–80 h (≈ 0.2–0.4 h per issue). **Doc:** this file. -### 7. OCR reads only unaccented Latin - -- **Missing:** every other script and accented Latin; editable-text output; deskew, rotation, - despeckle, background and camera clean-up; MRC compression; pluggable engines; OCR in the - browser; scanner input. -- **Evidence:** `ocr.ocr-languages` partial ("the models read only the Latin alphabet without - accents"); 19 of 26 OCR features planned; [#744](https://github.com/storytold/pdfcraft/issues/744) - (Dutch). Acrobat ships on-device recognition for dozens of languages. -- **Impact:** scanned documents are a core Pro workflow outside English. -- **Estimate:** 40–75 h, partly blocked on openly licensed models. **Doc:** - [file-format-parity.md](file-format-parity.md). - +### 7. OCR still has language and scan-cleanup gaps + +- **Now available:** English, German, French, Spanish and horizontal Japanese, Unicode search/copy, + automatic quarter-turn correction and deskew up to 12 degrees, including correction without OCR. +- **Missing:** additional language choices, vertical Japanese, editable-text output, deskew on pages + with annotations, despeckle, background/camera cleanup, MRC compression, browser OCR and scanner input. +- **Evidence:** `crates/engine/tests/scan_ocr.rs` and the OCR automation/UI tests; model tests require + `cargo xtask models`, Japanese fixture generation requires the release `CRAFT_FONTS_DIR` input. +- **Impact:** scanned documents remain a core Pro workflow with wider language and layout demands. +- **Estimate:** previous 40–75 h estimate needs reassessment after this narrower feature slice. ### 8. Office export weak, Office import absent - **Missing:** Word export that keeps layout; Excel and PowerPoint export; creating PDFs from Office, diff --git a/docs/roadmap.md b/docs/roadmap.md index aed24a3dbd..3373332e89 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # PdfCraft roadmap: milestones and what's next -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** minor (localized shortcut labels, #496; alpha gate added; convert-and-share re-judged with the cross-app rule → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (localized shortcut labels, #496; alpha gate added; convert-and-share re-judged with the cross-app rule → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) Forward-looking: the milestones, the current focus and what comes next. The parity numbers behind it are in [target-app-parity.md](target-app-parity.md), the ranked work list in [gaps.md](gaps.md), @@ -74,7 +74,7 @@ total matches the parity estimate. | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | | M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | -| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; Create PDF from multiple files (PDFs, images and text, edited as pages in the grid). Missing: OCR languages beyond Latin, editable-text OCR output, Office export/import, web printing | +| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; Create PDF from multiple files (PDFs, images and text, edited as pages in the grid). Done since: German/French/Spanish/horizontal-Japanese OCR, Unicode text layers, automatic quarter-turn correction, and bounded deskew through UI and automation. Missing: additional OCR languages, deskew with annotations, editable-text OCR output, Office export/import, web printing | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools).
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export. Missing: autotag, Tags/Order/Content panels, Reading Order tool, alt-text workflow, compare, geospatial/3D measurement, search index, XFA | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers | @@ -119,6 +119,8 @@ M2 (renderer), M7 (content editing), M10 (OCR and Office), M11 (Preflight and st ## Revision history +- 2026-10-11: M10 multilingual scan recognition and orientation/deskew correction; overall milestone percentage unchanged. + | Date | Change | Summary | |---|---|---| | 2026-10-10 | minor | M14: localized shortcut key names (#496), including runtime switching and native layout checks. Estimates unchanged. | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 2438bcc484..fdd0edd271 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6438,8 +6438,8 @@ milestone = "M10" status = "partial" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/ui-egui/tests/ocr.rs::recognize_text_dialog_adds_searchable_text"] -notes = "Language choice exists, but the models read only the Latin alphabet without accents (English)." +evidence = ["crates/engine/tests/scan_ocr.rs::scan_recognizes_accents_in_requested_languages", "crates/engine/tests/scan_ocr.rs::scan_recognizes_japanese_with_release_fonts"] +notes = "English, German, French, Spanish and horizontal Japanese run offline. Other language choices and vertical Japanese remain unclaimed." [[feature]] id = "ocr.searchable-image" @@ -6447,11 +6447,11 @@ title = "Searchable image output (invisible text layer)" area = "I" tier = "P0" milestone = "M10" -status = "partial" +status = "shipped" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable"] -notes = "Invisible text (Tr 3) fitted to each word box; the image is not deskewed or downsampled (exact mode only)." +evidence = ["crates/engine/tests/scan_ocr.rs::unicode_ocr_survives_save_search_and_undo_without_changing_pixels"] +notes = "Unicode invisible text with ToUnicode mappings; optional orientation and deskew correction preserves original image streams." [[feature]] id = "ocr.searchable-image-exact" @@ -6462,8 +6462,8 @@ milestone = "M10" status = "shipped" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable"] -notes = "The image is left untouched." +evidence = ["crates/engine/tests/scan_ocr.rs::unicode_ocr_survives_save_search_and_undo_without_changing_pixels"] +notes = "Disable auto_rotate and deskew for unchanged page pixels; OCR does not recompress the original image." [[feature]] id = "ocr.editable-text-output" @@ -6507,11 +6507,11 @@ title = "Detect and fix page rotation" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "shipped" +commands = ["ocr.recognize", "ocr.enhance"] +tools = ["ocr_recognize", "ocr_enhance"] +evidence = ["crates/engine/tests/scan_ocr.rs::scan_correction_handles_all_quarter_turns_and_stale_results", "crates/automation/tests/automation.rs::ocr_enhance_corrects_rotation_without_adding_text"] +notes = "Confidence-gated 90/180/270 degree correction using PP-LCNet. Blank and uncertain pages stay unchanged; correction alone or before OCR, undoable." [[feature]] id = "ocr.deskew" @@ -6519,11 +6519,11 @@ title = "Deskew" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize", "ocr.enhance"] +tools = ["ocr_recognize", "ocr_enhance"] +evidence = ["crates/engine/tests/scan_ocr.rs::scan_deskew_without_ocr_persists_and_is_undoable"] +notes = "Projection-profile deskew up to 12 degrees, fitting the whole scan inside the page. Preserves original image streams; pages with annotations are refused to avoid misalignment. No perspective or camera dewarping." [[feature]] id = "ocr.despeckle" From 78a345652277e3699354ddd7b6974e915f4394f3 Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:14:38 +0200 Subject: [PATCH 02/19] M10: add Italian OCR and accented-text regression coverage Expose Italiano in the shared language list and derive both automation schemas from it. Cover all six Italian accented vowels through real-model OCR, save/reopen and unchanged scan pixels, plus UI selection and single/batch automation. Requested by @magdale76 in #1012. Validated with 2,147 passing tests including the corrected UI selector and the existing Windows update-test filename workaround; the existing symlink test requires unavailable privileges and remains skipped. Co-authored-by: Codex --- ROADMAP.md | 5 ++++- crates/automation/src/tools.rs | 9 +++++++-- crates/automation/tests/automation.rs | 12 +++++++++--- crates/engine/tests/scan_ocr.rs | 18 ++++++++++++++++++ crates/ocr/README.md | 6 +++++- crates/ocr/src/lib.rs | 3 ++- crates/ui-egui/tests/ocr.rs | 5 +++++ docs/roadmap.md | 6 ++++-- parity/acrobat-features.toml | 2 +- 9 files changed, 55 insertions(+), 11 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index d565c57d7e..cc89bd49ac 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ **Stage: alpha** · next: beta, ~26% (ready for real work 49% → 75%) and ~400–700 h away -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** minor (localized shortcut labels, #496; readiness table with hours per audience; full number as weighted sum 49%; earlier: second look: ready 40% → 47%, mainstream 55% added; core-workflow gate applied → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (Italian OCR; localized shortcut labels, #496; readiness table with hours per audience; full number as weighted sum 49%; earlier: second look: ready 40% → 47%, mainstream 55% added; core-workflow gate applied → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) PdfCraft's one-page summary: how close it is to Adobe Acrobat Pro's offline feature set, and what comes next. Adobe's cloud services (Document Cloud, Acrobat Sign requests, shared review, the AI @@ -133,6 +133,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-11 (M10, Italian OCR):** Added Italiano (it) to recognition in the UI, CLI and MCP. Both OCR schemas use the engine language list. A real-model regression checks all six Italian accented vowels after save/reopen and unchanged scan pixels; UI and single/batch automation cover selection. Overall percentage unchanged. + - **2026-10-11 (M10, paper-edge crop):** Enhance scanned file crops black/white scanner borders at clear paper edges with adjustable safety padding, preserving blank notebook margins and colored covers. Detection runs before rotation/deskew/OCR; original image streams survive and all changes undo together. Engine, automation and UI tests cover rotated/off-origin pages, padding, save/reopen and OCR geometry. Indistinguishable white-on-white borders stay unchanged. Overall percentage unchanged. - **2026-10-11 (M10, scan correction):** Offline OCR adds German, French, Spanish and horizontal Japanese with Unicode search/copy, confidence-gated quarter-turn correction, and deskew up to 12 degrees. Correction can run without OCR through the UI and automation; original image streams survive, changes undo together, and stale results are refused. Deskew on annotated pages, vertical Japanese, dewarping and editable-text output remain open. Overall percentage unchanged. @@ -320,6 +322,7 @@ Newest first: the date, what moved, and the new overall estimate. | Date | Change | Summary | |---|---|---| +| 2026-10-11 | minor | Italian OCR selection and accented-text save/reopen coverage; estimates unchanged | | 2026-10-10 | minor | Readiness table with hours per audience; full number restated as the additive weighted sum over dimensions (47% → 49%; method aligned with the standard, no new evidence) | | 2026-10-10 | minor | Essentials-user score added (≈ 66%) | | 2026-10-10 | minor | Gate consistency check: convert and share re-judged partial, not blocking; stage pre-alpha → alpha | diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4c5bb8c9f5..d1ae2e19c2 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -30,6 +30,11 @@ fn pages(what: &str) -> Value { json!({ "type": "array", "items": { "type": "integer", "minimum": 1 }, "minItems": 1, "description": format!("1-based page numbers {what}.") }) } +fn ocr_language() -> Value { + let codes: Vec<&str> = pdfcraft_engine::ocr::LANGUAGES.iter().map(|language| language.0).collect(); + json!({ "type": "string", "enum": codes, "description": "Document language (default en)." }) +} + fn path(what: &str) -> Value { json!({ "type": "array", "items": { "type": "integer", "minimum": 1 }, "description": format!("{what}: 1-based positions from the top level, e.g. [2, 1] = the first child of the second bookmark.") }) } @@ -924,7 +929,7 @@ pub fn tools() -> Vec { "doc": doc(), "pages": pages("to recognise (default: all)"), "dpi": { "type": "number", "minimum": 72, "maximum": 600, "description": "Resolution pages are read at (default 300; a scanned page is read at most at its own resolution)." }, - "auto_rotate": { "type": "boolean", "default": true }, "deskew": { "type": "boolean", "default": true }, "auto_crop": { "type": "boolean", "default": false }, "crop_margin": { "type": "number", "minimum": 0, "maximum": 20, "default": 2 }, "recognize": { "type": "boolean", "default": true }, "language": { "type": "string", "enum": ["en", "de", "ja", "fr", "es"], "description": "Document language (default en)." }, + "auto_rotate": { "type": "boolean", "default": true }, "deskew": { "type": "boolean", "default": true }, "auto_crop": { "type": "boolean", "default": false }, "crop_margin": { "type": "number", "minimum": 0, "maximum": 20, "default": 2 }, "recognize": { "type": "boolean", "default": true }, "language": ocr_language(), "skip_text_pages": { "type": "boolean", "description": "Leave pages that already contain text alone (default true)." }, }), &["doc"], @@ -936,7 +941,7 @@ pub fn tools() -> Vec { "paths": { "type": "array", "items": { "type": "string" }, "minItems": 1 }, "folder": { "type": "string" }, "dpi": { "type": "number", "minimum": 72, "maximum": 600 }, - "auto_rotate": { "type": "boolean", "default": true }, "deskew": { "type": "boolean", "default": true }, "auto_crop": { "type": "boolean", "default": false }, "crop_margin": { "type": "number", "minimum": 0, "maximum": 20, "default": 2 }, "recognize": { "type": "boolean", "default": true }, "language": { "type": "string", "enum": ["en", "de", "ja", "fr", "es"] }, + "auto_rotate": { "type": "boolean", "default": true }, "deskew": { "type": "boolean", "default": true }, "auto_crop": { "type": "boolean", "default": false }, "crop_margin": { "type": "number", "minimum": 0, "maximum": 20, "default": 2 }, "recognize": { "type": "boolean", "default": true }, "language": ocr_language(), }), &["paths", "folder"], )), diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index e39527240e..135a50c757 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -2947,9 +2947,15 @@ fn ocr_tools_make_a_scan_searchable() { let mut a = auto(&dir); let status = ok(&mut a, "ocr_status", json!({})); assert_eq!(status["languages"][0]["code"], "en"); - for language in ["de", "ja", "fr", "es"] { + for language in ["de", "ja", "fr", "es", "it"] { assert!(status["languages"].as_array().unwrap().iter().any(|l| l["code"] == language)); } + for name in ["ocr_recognize", "ocr_recognize_files"] { + let tool = tools().into_iter().find(|tool| tool.name == name).unwrap(); + let advertised = &tool.input_schema["properties"]["language"]["enum"]; + let available: Vec<_> = status["languages"].as_array().unwrap().iter().map(|language| language["code"].clone()).collect(); + assert_eq!(advertised, &json!(available), "{name} schema must advertise the same languages as ocr_status"); + } if status["available"] != true { eprintln!("skipped: OCR models not installed"); return; @@ -2958,7 +2964,7 @@ fn ocr_tools_make_a_scan_searchable() { ok(&mut a, "doc_export_images", json!({ "doc": text, "folder": "scan", "dpi": 150, "pages": [1] })); let png = std::fs::read_dir(dir.join("scan")).unwrap().next().unwrap().unwrap().path(); let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": [png.to_string_lossy()] }))["doc"].as_u64().unwrap(); - let r = ok(&mut a, "ocr_recognize", json!({ "doc": scan })); + let r = ok(&mut a, "ocr_recognize", json!({ "doc": scan, "language": "it" })); assert!(r["pages"][0]["text"].as_str().unwrap().to_lowercase().contains("searchable"), "{r}"); assert!(r["words"].as_u64().unwrap() >= 4); let again = ok(&mut a, "ocr_recognize", json!({ "doc": scan, "pages": [1] })); @@ -3039,7 +3045,7 @@ fn ocr_recognize_files_writes_searchable_copies() { let png = std::fs::read_dir(dir.join("scan")).unwrap().next().unwrap().unwrap().path(); let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": [png.to_string_lossy()] }))["doc"].as_u64().unwrap(); ok(&mut a, "doc_save", json!({ "doc": scan, "path": "in/scan.pdf" })); - let r = ok(&mut a, "ocr_recognize_files", json!({ "paths": ["in/scan.pdf", "missing.pdf"], "folder": "out" })); + let r = ok(&mut a, "ocr_recognize_files", json!({ "paths": ["in/scan.pdf", "missing.pdf"], "folder": "out", "language": "it" })); assert!(r["files"][0]["words"].as_u64().unwrap() >= 3, "{r}"); assert!(r["files"][1]["error"].is_string(), "{r}"); let out = ok(&mut a, "doc_open", json!({ "path": "out/scan.pdf" }))["doc"].as_u64().unwrap(); diff --git a/crates/engine/tests/scan_ocr.rs b/crates/engine/tests/scan_ocr.rs index 6c229d8462..4033cd23de 100644 --- a/crates/engine/tests/scan_ocr.rs +++ b/crates/engine/tests/scan_ocr.rs @@ -210,6 +210,24 @@ fn scan_recognizes_accents_in_requested_languages() { } } +#[test] +fn italian_ocr_preserves_all_six_accents_after_save() { + if !ocr::available() { + eprintln!("skipped: run cargo xtask models"); + return; + } + let (mut s, id) = scan("Lunedì visiterò la città perché è più tranquilla.", 0, 0.0); + let before = render(&s, id); + s.recognize_text(id, &[], OcrSettings { language: "it".into(), auto_rotate: false, deskew: false, ..Default::default() }).unwrap(); + let saved = s.save_bytes(id).unwrap(); + let reopened = s.open("italian.pdf", None, saved, None).unwrap(); + let extracted = text(&s, reopened); + for expected in ["Lunedì", "visiterò", "città", "perché", "è", "più"] { + assert!(extracted.split_whitespace().any(|word| word == expected), "missing {expected:?}: {extracted:?}"); + } + assert_eq!(before, render(&s, reopened), "searchable text must not change the scan pixels"); +} + #[test] fn scan_recognizes_japanese_with_release_fonts() { if !ocr::available() || pdfcraft_fonts::CRAFT_FONTS.is_empty() { diff --git a/crates/ocr/README.md b/crates/ocr/README.md index 2cfbe95c6f..8810405c0b 100644 --- a/crates/ocr/README.md +++ b/crates/ocr/README.md @@ -4,7 +4,7 @@ `ATTRIBUTION.toml`. No document is uploaded and no model is downloaded at runtime. Release packages ship these files beside the application. -- English, German, French, Spanish and horizontal Japanese use PaddlePaddle's +- English, German, French, Spanish, Italian and horizontal Japanese use PaddlePaddle's PP-OCRv6 small model through the existing Rust RTen runtime. The legacy English recognizer API remains available to existing callers. - PP-LCNet classifies quarter-turn orientation. Preprocessing centres on ink bounds, @@ -42,3 +42,7 @@ languages, all quarter-turns, deskew, crop offsets and padding, unchanged exact- and stale results. Model tests report a skip when models are absent. The Japanese synthetic fixture uses the optional release `CRAFT_FONTS_DIR` checkout; generated fixtures and third-party document samples are not committed. + +The Italian regression recognizes all six accented vowels (`à è é ì ò ù`) in a +synthetic sentence and checks that they survive saving/reopening without changing +the scanned pixels. This is a regression fixture, not a real-document accuracy benchmark. diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index eeb4d86330..dc49cae554 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -27,7 +27,8 @@ pub const DICTIONARY: &str = "multilingual-dictionary.txt"; pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; /// Supported and tested language choices (ISO 639-1). -pub const LANGUAGES: &[(&str, &str)] = &[("en", "English"), ("de", "Deutsch"), ("ja", "日本語"), ("fr", "Français"), ("es", "Español")]; +pub const LANGUAGES: &[(&str, &str)] = + &[("en", "English"), ("de", "Deutsch"), ("ja", "日本語"), ("fr", "Français"), ("es", "Español"), ("it", "Italiano")]; #[derive(Debug, thiserror::Error)] pub enum OcrError { diff --git a/crates/ui-egui/tests/ocr.rs b/crates/ui-egui/tests/ocr.rs index c1c14264dc..6d2bb120b9 100644 --- a/crates/ui-egui/tests/ocr.rs +++ b/crates/ui-egui/tests/ocr.rs @@ -32,6 +32,11 @@ fn recognize_text_dialog_adds_searchable_text() { h.get_by_label("Automatically rotate sideways and upside-down pages"); h.get_by_label("Straighten skewed scans"); assert!(h.state().ocr_draft.auto_rotate && h.state().ocr_draft.deskew); + h.get_by_value("English").click(); + h.run_steps(2); + h.get_by_label("Italiano").click(); + h.run_steps(2); + assert_eq!(h.state().ocr_draft.language, "it"); if !pdfcraft_engine::ocr::available() { eprintln!("skipped: OCR models not installed"); return; diff --git a/docs/roadmap.md b/docs/roadmap.md index 4d13832967..6ab58b6fca 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # PdfCraft roadmap: milestones and what's next -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (localized shortcut labels, #496; alpha gate added; convert-and-share re-judged with the cross-app rule → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (Italian OCR; localized shortcut labels, #496; alpha gate added; convert-and-share re-judged with the cross-app rule → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) Forward-looking: the milestones, the current focus and what comes next. The parity numbers behind it are in [target-app-parity.md](target-app-parity.md), the ranked work list in [gaps.md](gaps.md), @@ -74,7 +74,7 @@ total matches the parity estimate. | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | | M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | -| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; Create PDF from multiple files (PDFs, images and text, edited as pages in the grid). Done since: German/French/Spanish/horizontal-Japanese OCR, Unicode text layers, automatic quarter-turn correction, bounded deskew, and automatic paper-edge crop with safety padding through UI and automation. Missing: additional OCR languages, deskew with annotations, editable-text OCR output, Office export/import, web printing | +| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; Create PDF from multiple files (PDFs, images and text, edited as pages in the grid). Done since: German/French/Spanish/Italian/horizontal-Japanese OCR, Unicode text layers, automatic quarter-turn correction, bounded deskew, and automatic paper-edge crop with safety padding through UI and automation. Missing: additional OCR languages, deskew with annotations, editable-text OCR output, Office export/import, web printing | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools).
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export. Missing: autotag, Tags/Order/Content panels, Reading Order tool, alt-text workflow, compare, geospatial/3D measurement, search index, XFA | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers | @@ -119,6 +119,8 @@ M2 (renderer), M7 (content editing), M10 (OCR and Office), M11 (Preflight and st ## Revision history +- 2026-10-11: M10 Italian OCR selection and all six Italian accents covered by a real-model save/reopen regression; overall milestone percentage unchanged. + - 2026-10-11: M10 multilingual scan recognition and orientation/deskew correction; overall milestone percentage unchanged. - 2026-10-11: M10 automatic scanner-border crop at clear paper edges, adjustable padding, and shared OCR/rotation/deskew undo. Indistinguishable white edges remain open; percentage unchanged. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 62e7e1519d..06e5142f62 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6439,7 +6439,7 @@ status = "partial" commands = ["ocr.recognize"] tools = ["ocr_recognize"] evidence = ["crates/engine/tests/scan_ocr.rs::scan_recognizes_accents_in_requested_languages", "crates/engine/tests/scan_ocr.rs::scan_recognizes_japanese_with_release_fonts"] -notes = "English, German, French, Spanish and horizontal Japanese run offline. Other language choices and vertical Japanese remain unclaimed." +notes = "English, German, French, Spanish, Italian and horizontal Japanese run offline. Other language choices and vertical Japanese remain unclaimed." [[feature]] id = "ocr.searchable-image" From 5ea94bad7b5d9c77a8eb86414376cfb6b7efe4f8 Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:30:08 +0200 Subject: [PATCH 03/19] M10: add Latin OCR languages and system-language defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold Dutch, Danish, Norwegian Bokmål, Swedish, Finnish and Portuguese into the existing OCR language list following @mlkonrad's comparison in #1012. Keep the shared model unchanged; leave Icelandic unclaimed. Add real-model save/reopen checks for accented words in all six languages. Default the OCR picker to the primary OS/browser locale, independently of available UI translations, with English fallback. Preserve manual document-language choices and deterministic automation defaults. Cover locale variants, unsupported locales and UI selection. Validation: 2,149 tests passed; seven pre-existing ignored and one Windows symlink-privilege test skipped. Four update tests passed under a neutral executable filename due to Windows installer detection. Format, strict workspace Clippy, 30 WASM crates, layers, assets and parity passed. German OCR dialog rendered and inspected. Co-authored-by: Codex --- Cargo.lock | 3 ++ ROADMAP.md | 5 ++- crates/automation/tests/automation.rs | 2 +- crates/engine/src/ocr.rs | 2 +- crates/engine/tests/scan_ocr.rs | 31 ++++++++++++++ crates/ocr/README.md | 13 +++++- crates/ocr/src/lib.rs | 60 ++++++++++++++++++++++++++- crates/ui-egui/Cargo.toml | 6 +-- crates/ui-egui/src/ocr_ui.rs | 2 +- crates/ui-egui/tests/ocr.rs | 9 +++- docs/roadmap.md | 6 ++- parity/acrobat-features.toml | 4 +- 12 files changed, 127 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef1138665a..56ae4c4127 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5491,7 +5491,10 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" dependencies = [ + "js-sys", "libc", + "wasm-bindgen", + "web-sys", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index cc89bd49ac..8d32b0dd98 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ **Stage: alpha** · next: beta, ~26% (ready for real work 49% → 75%) and ~400–700 h away -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (Italian OCR; localized shortcut labels, #496; readiness table with hours per audience; full number as weighted sum 49%; earlier: second look: ready 40% → 47%, mainstream 55% added; core-workflow gate applied → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (additional OCR languages and system-language default; localized shortcut labels, #496; readiness table with hours per audience; full number as weighted sum 49%; earlier: second look: ready 40% → 47%, mainstream 55% added; core-workflow gate applied → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) PdfCraft's one-page summary: how close it is to Adobe Acrobat Pro's offline feature set, and what comes next. Adobe's cloud services (Document Cloud, Acrobat Sign requests, shared review, the AI @@ -133,6 +133,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-11 (M10, OCR languages and defaults):** Fold Dutch, Danish, Norwegian Bokmål, Swedish, Finnish and Portuguese into the scan/OCR work, with save/reopen regressions for accented words. The UI selects the primary system/browser language, independently of UI translations, with English fallback; manual selection remains available. Icelandic stays unclaimed. Overall percentage unchanged. + - **2026-10-11 (M10, Italian OCR):** Added Italiano (it) to recognition in the UI, CLI and MCP. Both OCR schemas use the engine language list. A real-model regression checks all six Italian accented vowels after save/reopen and unchanged scan pixels; UI and single/batch automation cover selection. Overall percentage unchanged. - **2026-10-11 (M10, paper-edge crop):** Enhance scanned file crops black/white scanner borders at clear paper edges with adjustable safety padding, preserving blank notebook margins and colored covers. Detection runs before rotation/deskew/OCR; original image streams survive and all changes undo together. Engine, automation and UI tests cover rotated/off-origin pages, padding, save/reopen and OCR geometry. Indistinguishable white-on-white borders stay unchanged. Overall percentage unchanged. @@ -322,6 +324,7 @@ Newest first: the date, what moved, and the new overall estimate. | Date | Change | Summary | |---|---|---| +| 2026-10-11 | minor | Additional Latin OCR languages and system-language default with English fallback; estimates unchanged | | 2026-10-11 | minor | Italian OCR selection and accented-text save/reopen coverage; estimates unchanged | | 2026-10-10 | minor | Readiness table with hours per audience; full number restated as the additive weighted sum over dimensions (47% → 49%; method aligned with the standard, no new evidence) | | 2026-10-10 | minor | Essentials-user score added (≈ 66%) | diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 135a50c757..9bcdd0c8f0 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -2947,7 +2947,7 @@ fn ocr_tools_make_a_scan_searchable() { let mut a = auto(&dir); let status = ok(&mut a, "ocr_status", json!({})); assert_eq!(status["languages"][0]["code"], "en"); - for language in ["de", "ja", "fr", "es", "it"] { + for language in ["de", "ja", "fr", "es", "it", "nl", "da", "nb", "sv", "fi", "pt"] { assert!(status["languages"].as_array().unwrap().iter().any(|l| l["code"] == language)); } for name in ["ocr_recognize", "ocr_recognize_files"] { diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs index b693999490..4eaf0fc450 100644 --- a/crates/engine/src/ocr.rs +++ b/crates/engine/src/ocr.rs @@ -10,7 +10,7 @@ use std::sync::{Arc, Mutex}; use pdfcraft_render::{PageInfo, PageRenderer, RenderConfig, RenderRequest, RequestKind}; pub use pdfcraft_ocr::scan::Correction; -pub use pdfcraft_ocr::{LANGUAGES, Models, Ocr, OcrError, PlacedWord}; +pub use pdfcraft_ocr::{LANGUAGES, Models, Ocr, OcrError, PlacedWord, language_for_locale}; use crate::{DocId, Edit, EditError, Session}; diff --git a/crates/engine/tests/scan_ocr.rs b/crates/engine/tests/scan_ocr.rs index 4033cd23de..bd657849f9 100644 --- a/crates/engine/tests/scan_ocr.rs +++ b/crates/engine/tests/scan_ocr.rs @@ -228,6 +228,37 @@ fn italian_ocr_preserves_all_six_accents_after_save() { assert_eq!(before, render(&s, reopened), "searchable text must not change the scan pixels"); } +#[test] +fn additional_latin_languages_keep_accented_words_after_save() { + if !ocr::available() { + eprintln!("skipped: run cargo xtask models"); + return; + } + let samples: &[(&str, &str, &[&str])] = &[ + ("nl", "Een coöperatie in België koopt één café.", &["coöperatie", "België", "één"]), + ("da", "Vi går på øen og læser bøger.", &["går", "på", "øen", "læser", "bøger"]), + ("nb", "Vi går på øya og leser bøker.", &["går", "på", "øya", "bøker"]), + ("sv", "Vi läser böcker på en ö nära staden.", &["läser", "böcker", "på", "ö", "nära"]), + ("fi", "Hyvää päivää ystävä. Täällä on lämmin kesä.", &["Hyvää", "päivää", "ystävä", "Täällä", "lämmin", "kesä"]), + ("pt", "A ação começa amanhã com café e música.", &["ação", "começa", "amanhã", "café", "música"]), + ]; + for (language, sentence, expected) in samples { + let (mut s, id) = scan(sentence, 0, 0.0); + let before = render(&s, id); + s.recognize_text(id, &[], OcrSettings { language: (*language).into(), auto_rotate: false, deskew: false, ..Default::default() }).unwrap(); + let saved = s.save_bytes(id).unwrap(); + let reopened = s.open("recognized.pdf", None, saved, None).unwrap(); + let extracted = text(&s, reopened); + for word in *expected { + assert!( + extracted.split_whitespace().any(|w| w.trim_matches(|c: char| !c.is_alphabetic()) == *word), + "{language} missing {word:?}: {extracted:?}" + ); + } + assert_eq!(before, render(&s, reopened), "{language}: OCR must preserve scan pixels"); + } +} + #[test] fn scan_recognizes_japanese_with_release_fonts() { if !ocr::available() || pdfcraft_fonts::CRAFT_FONTS.is_empty() { diff --git a/crates/ocr/README.md b/crates/ocr/README.md index 8810405c0b..501069ddcd 100644 --- a/crates/ocr/README.md +++ b/crates/ocr/README.md @@ -4,7 +4,8 @@ `ATTRIBUTION.toml`. No document is uploaded and no model is downloaded at runtime. Release packages ship these files beside the application. -- English, German, French, Spanish, Italian and horizontal Japanese use PaddlePaddle's +- English, German, French, Spanish, Italian, Dutch, Danish, Norwegian Bokmål, + Swedish, Finnish, Portuguese and horizontal Japanese use PaddlePaddle's PP-OCRv6 small model through the existing Rust RTen runtime. The legacy English recognizer API remains available to existing callers. - PP-LCNet classifies quarter-turn orientation. Preprocessing centres on ink bounds, @@ -32,6 +33,11 @@ default. Per-page `crop_box` results are in original PDF user space, before corr Corrections and OCR form one undo step; stale worker results and cancelled UI runs do not modify the document. +The UI initially selects the primary OS/browser language independently of its interface +translation. Regional locales such as `de-AT`, `nl-BE` and `pt-BR` use `de`, `nl` and +`pt`; unavailable or unsupported locales fall back to English. Users can select a +different document language. CLI/MCP calls retain English as their deterministic default. + Limits: no perspective correction/dewarping, no guarantee on photos or ambiguous text, no vertical Japanese claim, and deskew refuses annotated pages rather than separating annotations from their targets. Supported OCR languages are deliberately limited to @@ -46,3 +52,8 @@ fixtures and third-party document samples are not committed. The Italian regression recognizes all six accented vowels (`à è é ì ò ù`) in a synthetic sentence and checks that they survive saving/reopening without changing the scanned pixels. This is a regression fixture, not a real-document accuracy benchmark. + +The additional Latin-language regression checks Dutch `één`, `coöperatie`, `België` +and accented words in Danish, Norwegian, Swedish, Finnish and Portuguese after +save/reopen. The same model is used without per-language character restrictions. +Icelandic remains unclaimed because community tests found weak recognition of þ/Þ and Í. diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index dc49cae554..3a1f9142ac 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -27,8 +27,29 @@ pub const DICTIONARY: &str = "multilingual-dictionary.txt"; pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; /// Supported and tested language choices (ISO 639-1). -pub const LANGUAGES: &[(&str, &str)] = - &[("en", "English"), ("de", "Deutsch"), ("ja", "日本語"), ("fr", "Français"), ("es", "Español"), ("it", "Italiano")]; +pub const LANGUAGES: &[(&str, &str)] = &[ + ("en", "English"), + ("de", "Deutsch"), + ("ja", "日本語"), + ("fr", "Français"), + ("es", "Español"), + ("it", "Italiano"), + ("nl", "Nederlands"), + ("da", "Dansk"), + ("nb", "Norsk bokmål"), + ("sv", "Svenska"), + ("fi", "Suomi"), + ("pt", "Português"), +]; + +/// Map the primary system locale to an OCR choice, independently of UI translations. +/// Regional variants share a recognizer. Missing or unsupported locales use English. +pub fn language_for_locale(locale: Option<&str>) -> &'static str { + let primary = locale.unwrap_or_default().trim().split(['-', '_', '.', '@']).next().unwrap_or_default(); + // Older platforms use the generic Norwegian tag for Bokmål. + let primary = if primary.eq_ignore_ascii_case("no") { "nb" } else { primary }; + LANGUAGES.iter().find(|(code, _)| code.eq_ignore_ascii_case(primary)).map_or("en", |(code, _)| *code) +} #[derive(Debug, thiserror::Error)] pub enum OcrError { @@ -258,6 +279,41 @@ pub fn text_layer(words: &[PlacedWord]) -> Vec { #[cfg(test)] mod tests { + #[test] + fn system_locale_selects_ocr_language_with_english_fallback() { + for (locale, expected) in [ + ("de-AT", "de"), + ("de_CH.UTF-8", "de"), + ("JA-jp", "ja"), + ("fr-CA", "fr"), + ("es-MX", "es"), + ("it-IT", "it"), + ("nl-BE", "nl"), + ("da-DK", "da"), + ("nb-NO", "nb"), + ("no_NO", "nb"), + ("sv-FI", "sv"), + ("fi_FI", "fi"), + ("pt-BR", "pt"), + ("pt_PT@euro", "pt"), + (" EN-us ", "en"), + ("is-IS", "en"), + ("nn-NO", "en"), + ("zh-Hans", "en"), + ("C.UTF-8", "en"), + ("POSIX", "en"), + ("", "en"), + ("日本語", "en"), + ("deutsch", "en"), + ] { + assert_eq!(super::language_for_locale(Some(locale)), expected, "{locale}"); + } + assert_eq!(super::language_for_locale(None), "en"); + for (code, _) in super::LANGUAGES { + assert_eq!(super::language_for_locale(Some(code)), *code); + } + } + use super::*; #[test] diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 713e5b2a4b..6ce7da41be 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -30,6 +30,8 @@ png = "0.18" base64 = "0.22" # Right-to-left file names in the chrome (egui shapes but does not reorder); already in the tree. unicode-bidi = "0.3" +# OCR defaults to the OS/browser locale, including languages without a UI translation. +sys-locale = { version = "0.3", features = ["js"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Control-channel tokens. @@ -48,10 +50,6 @@ objc2 = "0.6" objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "NSEvent", "NSScreen"] } objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSArray", "NSGeometry"] } -[target.'cfg(windows)'.dependencies] -# Auto interface language: the Windows display language, without starting a process. -sys-locale = "0.3" - [target.'cfg(target_arch = "wasm32")'.dependencies] rfd = { workspace = true, features = ["file-handle-inner"] } wasm-bindgen-futures = "0.4" diff --git a/crates/ui-egui/src/ocr_ui.rs b/crates/ui-egui/src/ocr_ui.rs index 84b29c91a4..862cb18174 100644 --- a/crates/ui-egui/src/ocr_ui.rs +++ b/crates/ui-egui/src/ocr_ui.rs @@ -41,7 +41,7 @@ impl Default for OcrDraft { pages: OcrPages::All, from: 1, to: 1, - language: "en".into(), + language: pdfcraft_engine::ocr::language_for_locale(sys_locale::get_locale().as_deref()).into(), dpi: 300, auto_rotate: true, deskew: true, diff --git a/crates/ui-egui/tests/ocr.rs b/crates/ui-egui/tests/ocr.rs index 6d2bb120b9..2bbf981bbc 100644 --- a/crates/ui-egui/tests/ocr.rs +++ b/crates/ui-egui/tests/ocr.rs @@ -32,8 +32,14 @@ fn recognize_text_dialog_adds_searchable_text() { h.get_by_label("Automatically rotate sideways and upside-down pages"); h.get_by_label("Straighten skewed scans"); assert!(h.state().ocr_draft.auto_rotate && h.state().ocr_draft.deskew); - h.get_by_value("English").click(); + let expected = pdfcraft_engine::ocr::language_for_locale(sys_locale::get_locale().as_deref()); + assert_eq!(h.state().ocr_draft.language, expected, "OCR follows the system even when the interface is English"); + let label = pdfcraft_engine::ocr::LANGUAGES.iter().find(|(code, _)| *code == expected).unwrap().1; + h.get_by_value(label).click(); h.run_steps(2); + for label in ["Nederlands", "Dansk", "Norsk bokmål", "Svenska", "Suomi", "Português"] { + h.get_by_label(label); + } h.get_by_label("Italiano").click(); h.run_steps(2); assert_eq!(h.state().ocr_draft.language, "it"); @@ -65,6 +71,7 @@ fn enhance_scan_dialog_offers_correction_without_recognition() { h.get_by_label("Correct scans"); assert!(!h.state().ocr_draft.recognize); assert!(h.state().ocr_draft.auto_crop); + assert_eq!(h.state().ocr_draft.language, pdfcraft_engine::ocr::language_for_locale(sys_locale::get_locale().as_deref())); h.get_by_label("Safety margin (pt)"); h.get_by_label("Crop scanner borders to the paper edge").click(); h.run_steps(2); diff --git a/docs/roadmap.md b/docs/roadmap.md index 6ab58b6fca..8c2ef7eef0 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # PdfCraft roadmap: milestones and what's next -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (Italian OCR; localized shortcut labels, #496; alpha gate added; convert-and-share re-judged with the cross-app rule → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (additional OCR languages and system-language default; localized shortcut labels, #496; alpha gate added; convert-and-share re-judged with the cross-app rule → alpha) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) Forward-looking: the milestones, the current focus and what comes next. The parity numbers behind it are in [target-app-parity.md](target-app-parity.md), the ranked work list in [gaps.md](gaps.md), @@ -74,7 +74,7 @@ total matches the parity estimate. | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | | M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | -| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; Create PDF from multiple files (PDFs, images and text, edited as pages in the grid). Done since: German/French/Spanish/Italian/horizontal-Japanese OCR, Unicode text layers, automatic quarter-turn correction, bounded deskew, and automatic paper-edge crop with safety padding through UI and automation. Missing: additional OCR languages, deskew with annotations, editable-text OCR output, Office export/import, web printing | +| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; Create PDF from multiple files (PDFs, images and text, edited as pages in the grid). Done since: German/French/Spanish/Italian/Dutch/Danish/Norwegian/Swedish/Finnish/Portuguese/horizontal-Japanese OCR and system-language UI defaults, Unicode text layers, automatic quarter-turn correction, bounded deskew, and automatic paper-edge crop with safety padding through UI and automation. Missing: additional OCR languages, deskew with annotations, editable-text OCR output, Office export/import, web printing | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools).
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export. Missing: autotag, Tags/Order/Content panels, Reading Order tool, alt-text workflow, compare, geospatial/3D measurement, search index, XFA | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers | @@ -119,6 +119,8 @@ M2 (renderer), M7 (content editing), M10 (OCR and Office), M11 (Preflight and st ## Revision history +- 2026-10-11: M10 six more Latin OCR languages, accented-text regressions, and system/browser-language defaults with English fallback; percentages unchanged. + - 2026-10-11: M10 Italian OCR selection and all six Italian accents covered by a real-model save/reopen regression; overall milestone percentage unchanged. - 2026-10-11: M10 multilingual scan recognition and orientation/deskew correction; overall milestone percentage unchanged. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 06e5142f62..b9e120f4d4 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6438,8 +6438,8 @@ milestone = "M10" status = "partial" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/engine/tests/scan_ocr.rs::scan_recognizes_accents_in_requested_languages", "crates/engine/tests/scan_ocr.rs::scan_recognizes_japanese_with_release_fonts"] -notes = "English, German, French, Spanish, Italian and horizontal Japanese run offline. Other language choices and vertical Japanese remain unclaimed." +evidence = ["crates/engine/tests/scan_ocr.rs::scan_recognizes_accents_in_requested_languages", "crates/engine/tests/scan_ocr.rs::scan_recognizes_japanese_with_release_fonts", "crates/engine/tests/scan_ocr.rs::additional_latin_languages_keep_accented_words_after_save", "crates/ocr/src/lib.rs::system_locale_selects_ocr_language_with_english_fallback", "crates/ui-egui/tests/ocr.rs::recognize_text_dialog_adds_searchable_text"] +notes = "English, German, French, Spanish, Italian, Dutch, Danish, Norwegian Bokmål, Swedish, Finnish, Portuguese and horizontal Japanese run offline. The UI initially selects the primary system/browser locale, with English fallback when unavailable or unsupported; manual choices remain available. CLI/MCP language arguments retain their deterministic English default. Icelandic, other language choices and vertical Japanese remain unclaimed." [[feature]] id = "ocr.searchable-image" From f807821e5bbc658cc37c7102e2acc24d176bd56a Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:20:07 +0200 Subject: [PATCH 04/19] M10: add optional cross-platform Hot Folder processing service Add scheduled OCR tasks, durable source/output recovery, seven task-manager languages and optional boot services with platform installers. Co-authored-by: Codex --- .github/workflows/ci.yml | 14 +- .github/workflows/hot-folder-services.yml | 53 + .github/workflows/packaging-lint.yml | 4 +- .github/workflows/release.yml | 16 +- Cargo.lock | 902 +++++++++- ROADMAP.md | 2 + apps/pdfcraft-hot-folder/Cargo.toml | 28 + apps/pdfcraft-hot-folder/src/i18n.rs | 1041 +++++++++++ apps/pdfcraft-hot-folder/src/main.rs | 139 ++ apps/pdfcraft-hot-folder/src/source_ui.rs | 307 ++++ apps/pdfcraft-hot-folder/src/startup.rs | 10 + apps/pdfcraft-hot-folder/src/ui.rs | 1030 +++++++++++ apps/pdfcraft-hot-folder/src/worker.rs | 61 + crates/automation/Cargo.toml | 7 + crates/automation/README.md | 12 + crates/automation/src/hot_folder.rs | 150 ++ crates/automation/src/hot_folder_service.rs | 51 + crates/automation/src/lib.rs | 8 + crates/automation/src/tools.rs | 17 + crates/automation/tests/hot_folder.rs | 163 ++ crates/engine/Cargo.toml | 25 + crates/engine/src/hot_folder.rs | 1576 +++++++++++++++++ crates/engine/src/hot_folder_analysis.rs | 601 +++++++ crates/engine/src/hot_folder_output.rs | 735 ++++++++ crates/engine/src/hot_folder_preprocess.rs | 299 ++++ crates/engine/src/hot_folder_schedule.rs | 97 + crates/engine/src/hot_folder_sources.rs | 786 ++++++++ .../src/hot_folder_sources/credentials.rs | 618 +++++++ crates/engine/src/hot_folder_sources/ftp.rs | 704 ++++++++ .../engine/src/hot_folder_sources/outlook.rs | 786 ++++++++ .../engine/src/hot_folder_sources/project.rs | 211 +++ crates/engine/src/lib.rs | 12 + crates/engine/src/ocr.rs | 2 + crates/export/src/lib.rs | 3 + crates/export/src/publications.rs | 444 +++++ crates/hot-folder-service/Cargo.toml | 19 + .../examples/service_probe.rs | 75 + crates/hot-folder-service/src/definitions.rs | 134 ++ crates/hot-folder-service/src/lib.rs | 111 ++ crates/hot-folder-service/src/unix.rs | 203 +++ crates/hot-folder-service/src/unsupported.rs | 27 + crates/hot-folder-service/src/windows.rs | 157 ++ docs/hot-folder-analysis.md | 57 + docs/hot-folder-installation.md | 105 ++ docs/hot-folder-sources.md | 232 +++ docs/hot-folder.md | 154 ++ docs/roadmap.md | 2 +- packaging/linux/hot-folder-nfpm.yaml | 39 + packaging/linux/hot-folder-postinst.sh | 13 + packaging/linux/hot-folder-prerm.sh | 8 + packaging/linux/hot-folder.desktop | 8 + packaging/linux/nfpm.yaml | 2 + packaging/linux/package.sh | 16 +- packaging/macos/Distribution.xml.in | 16 + packaging/macos/HotFolder-Info.plist.in | 21 + .../macos/hot-folder-scripts/postinstall | 16 + packaging/macos/hot-folder-scripts/preinstall | 6 + packaging/macos/import-cert.sh | 19 +- packaging/macos/package.sh | 54 +- packaging/service-tests/linux-deb.sh | 37 + packaging/service-tests/macos-pkg.sh | 46 + packaging/service-tests/unix.sh | 41 + packaging/service-tests/windows-msi.ps1 | 57 + packaging/service-tests/windows.ps1 | 34 + packaging/windows/installer-ui.wxs | 34 +- packaging/windows/package.ps1 | 36 +- packaging/windows/pdfcraft.wxs | 60 +- packaging/windows/test-msi.ps1 | 29 + parity/acrobat-features.toml | 12 + xtask/src/layers.rs | 3 + 70 files changed, 12683 insertions(+), 114 deletions(-) create mode 100644 .github/workflows/hot-folder-services.yml create mode 100644 apps/pdfcraft-hot-folder/Cargo.toml create mode 100644 apps/pdfcraft-hot-folder/src/i18n.rs create mode 100644 apps/pdfcraft-hot-folder/src/main.rs create mode 100644 apps/pdfcraft-hot-folder/src/source_ui.rs create mode 100644 apps/pdfcraft-hot-folder/src/startup.rs create mode 100644 apps/pdfcraft-hot-folder/src/ui.rs create mode 100644 apps/pdfcraft-hot-folder/src/worker.rs create mode 100644 crates/automation/src/hot_folder.rs create mode 100644 crates/automation/src/hot_folder_service.rs create mode 100644 crates/automation/tests/hot_folder.rs create mode 100644 crates/engine/src/hot_folder.rs create mode 100644 crates/engine/src/hot_folder_analysis.rs create mode 100644 crates/engine/src/hot_folder_output.rs create mode 100644 crates/engine/src/hot_folder_preprocess.rs create mode 100644 crates/engine/src/hot_folder_schedule.rs create mode 100644 crates/engine/src/hot_folder_sources.rs create mode 100644 crates/engine/src/hot_folder_sources/credentials.rs create mode 100644 crates/engine/src/hot_folder_sources/ftp.rs create mode 100644 crates/engine/src/hot_folder_sources/outlook.rs create mode 100644 crates/engine/src/hot_folder_sources/project.rs create mode 100644 crates/export/src/publications.rs create mode 100644 crates/hot-folder-service/Cargo.toml create mode 100644 crates/hot-folder-service/examples/service_probe.rs create mode 100644 crates/hot-folder-service/src/definitions.rs create mode 100644 crates/hot-folder-service/src/lib.rs create mode 100644 crates/hot-folder-service/src/unix.rs create mode 100644 crates/hot-folder-service/src/unsupported.rs create mode 100644 crates/hot-folder-service/src/windows.rs create mode 100644 docs/hot-folder-analysis.md create mode 100644 docs/hot-folder-installation.md create mode 100644 docs/hot-folder-sources.md create mode 100644 docs/hot-folder.md create mode 100644 packaging/linux/hot-folder-nfpm.yaml create mode 100755 packaging/linux/hot-folder-postinst.sh create mode 100755 packaging/linux/hot-folder-prerm.sh create mode 100644 packaging/linux/hot-folder.desktop create mode 100644 packaging/macos/Distribution.xml.in create mode 100644 packaging/macos/HotFolder-Info.plist.in create mode 100755 packaging/macos/hot-folder-scripts/postinstall create mode 100755 packaging/macos/hot-folder-scripts/preinstall create mode 100755 packaging/service-tests/linux-deb.sh create mode 100755 packaging/service-tests/macos-pkg.sh create mode 100755 packaging/service-tests/unix.sh create mode 100644 packaging/service-tests/windows-msi.ps1 create mode 100644 packaging/service-tests/windows.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0c0596447..c10a3bb32b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev \ - libgl1-mesa-dev libgtk-3-dev + libgl1-mesa-dev libgtk-3-dev libdbus-1-dev - uses: dtolnay/rust-toolchain@stable with: @@ -79,6 +79,18 @@ jobs: - name: Layering run: cargo xtask layers + - name: Verify administrative credential provisioning (Linux) + if: runner.os == 'Linux' + shell: bash + env: + CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts + CRAFT_FONTS_REQUIRED: "1" + run: | + cargo test -p pdfcraft-engine --lib --no-run --message-format=json > "$RUNNER_TEMP/engine-tests.json" + test_binary="$(jq -rs '[.[] | select(.reason == "compiler-artifact" and .profile.test == true and .target.name == "pdfcraft_engine") | .executable | select(. != null)] | last // empty' "$RUNNER_TEMP/engine-tests.json")" + test -n "$test_binary" + sudo "$test_binary" --exact hot_folder_sources::credentials::tests::unix_admin_provisioning_preserves_service_uid_gid_and_private_modes --ignored + - name: Assets (licences and attribution) if: runner.os == 'Linux' run: cargo xtask assets diff --git a/.github/workflows/hot-folder-services.yml b/.github/workflows/hot-folder-services.yml new file mode 100644 index 0000000000..0de839d267 --- /dev/null +++ b/.github/workflows/hot-folder-services.yml @@ -0,0 +1,53 @@ +name: Hot Folder system services +on: + workflow_dispatch: + pull_request: + paths: + - 'crates/hot-folder-service/**' + - 'packaging/service-tests/**' + - 'packaging/windows/**' + - 'packaging/macos/**' + - 'packaging/linux/**' + - '.github/workflows/hot-folder-services.yml' +permissions: + contents: read +jobs: + lifecycle: + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2025, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - run: cargo fmt -p pdfcraft-hot-folder-service --check + - run: cargo clippy -p pdfcraft-hot-folder-service --all-targets -- -D warnings + - run: cargo test -p pdfcraft-hot-folder-service --all-targets + - run: cargo build -p pdfcraft-hot-folder-service --example service_probe + - if: runner.os != 'Windows' + run: bash packaging/service-tests/unix.sh + - if: runner.os == 'Windows' + shell: pwsh + run: packaging/service-tests/windows.ps1 + - if: runner.os == 'Windows' + run: dotnet tool install --global wix --version 5.0.2 + - if: runner.os == 'Windows' + shell: pwsh + run: packaging/service-tests/windows-msi.ps1 + - if: runner.os == 'macOS' + run: bash packaging/service-tests/macos-pkg.sh + - if: runner.os == 'Linux' + run: | + curl -fsSL -o "$RUNNER_TEMP/nfpm.deb" https://github.com/goreleaser/nfpm/releases/download/v2.47.0/nfpm_2.47.0_amd64.deb + sudo dpkg -i "$RUNNER_TEMP/nfpm.deb" + bash packaging/service-tests/linux-deb.sh + - if: always() && runner.os == 'Windows' + uses: actions/upload-artifact@v4 + with: + name: hot-folder-msi-lifecycle-logs + path: ${{ runner.temp }}/hot-folder-msi/*.log + if-no-files-found: ignore diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index aed79a90ed..01277cf8dd 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -34,7 +34,7 @@ jobs: run: | "$RUNNER_TEMP/actionlint" -color - name: shellcheck - run: shellcheck packaging/*.sh packaging/*/*.sh + run: shellcheck packaging/*.sh packaging/*/*.sh packaging/macos/hot-folder-scripts/* - name: PowerShell shell: pwsh run: | @@ -52,6 +52,8 @@ jobs: echo "::error file=packaging/windows/pdfcraft.wxs::Icon Id must end in .ico or .exe"; exit 1 fi desktop-file-validate packaging/linux/ai.storyteller.pdfcraft.desktop + desktop-file-validate packaging/linux/hot-folder.desktop + xmllint --noout packaging/macos/Distribution.xml.in sed -e 's/@VERSION@/0.0.0/' -e 's/@DATE@/2026-01-01/' packaging/linux/ai.storyteller.pdfcraft.metainfo.xml.in >"$RUNNER_TEMP/ai.storyteller.pdfcraft.metainfo.xml" appstreamcli validate --no-net --explain "$RUNNER_TEMP/ai.storyteller.pdfcraft.metainfo.xml" python3 - <<'EOF' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index af34d040d7..6655bb7d33 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -92,6 +92,8 @@ jobs: env: APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + APPLE_INSTALLER_CERTIFICATE: ${{ secrets.APPLE_INSTALLER_CERTIFICATE }} + APPLE_INSTALLER_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_INSTALLER_CERTIFICATE_PASSWORD }} KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} run: packaging/macos/import-cert.sh - name: Build, sign, notarize @@ -202,7 +204,7 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev \ - desktop-file-utils appstream python3-yaml zsync + desktop-file-utils appstream python3-yaml zsync libdbus-1-dev curl -fsSL -o "$RUNNER_TEMP/nfpm.deb" \ "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_${{ matrix.deb }}.deb" sudo dpkg -i "$RUNNER_TEMP/nfpm.deb" @@ -220,10 +222,14 @@ jobs: run: packaging/linux/package.sh - name: Check the packages run: | - dpkg-deb --info dist/release/*.deb - dpkg-deb --contents dist/release/*.deb | grep -E 'usr/bin/pdfcraft$|metainfo|applications' + main_deb="dist/release/pdfcraft-$PDFCRAFT_VERSION-linux-${{ matrix.arch }}.deb" + hot_deb="dist/release/pdfcraft-hot-folder-$PDFCRAFT_VERSION-linux-${{ matrix.arch }}.deb" + dpkg-deb --info "$main_deb" + dpkg-deb --info "$hot_deb" + dpkg-deb --contents "$main_deb" | grep -E 'usr/bin/pdfcraft$|metainfo|applications' + dpkg-deb --contents "$hot_deb" | grep -E 'usr/bin/pdfcraft-hot-folder$' # The OCR models (#103), where the installed app looks: /../share/pdfcraft/models. - dpkg-deb --contents dist/release/*.deb | grep -E 'usr/share/pdfcraft/models/.+\.rten$' + dpkg-deb --contents "$main_deb" | grep -E 'usr/share/pdfcraft/models/.+\.rten$' # Named exactly: dist/release also holds the CLI-only tar.gz, and `tar -tzf a b` reads b as a member. tar -tzf "dist/release/pdfcraft-$PDFCRAFT_VERSION-linux-${{ matrix.arch }}.tar.gz" | grep -E '/share/pdfcraft/models/.+\.rten$' ldd target/release/pdfcraft @@ -299,7 +305,7 @@ jobs: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu binutils-riscv64-linux-gnu libc6-dev-riscv64-cross \ libc6-dev:riscv64 qemu-user \ libxkbcommon-dev:riscv64 libwayland-dev:riscv64 libx11-dev:riscv64 libxrandr-dev:riscv64 \ - libxi-dev:riscv64 libgl1-mesa-dev:riscv64 + libxi-dev:riscv64 libgl1-mesa-dev:riscv64 libdbus-1-dev:riscv64 - name: Build and package (tar.gz, CLI tar.gz) env: CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts diff --git a/Cargo.lock b/Cargo.lock index 56ae4c4127..3e0b6e657c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ dependencies = [ "atspi-common", "phf 0.13.1", "serde", - "zvariant", + "zvariant 5.15.0", ] [[package]] @@ -86,7 +86,7 @@ dependencies = [ "futures-lite", "futures-util", "serde", - "zbus", + "zbus 5.19.0", ] [[package]] @@ -123,15 +123,26 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + [[package]] name = "aes" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ - "cipher", + "cipher 0.5.2", "cpubits", - "cpufeatures", + "cpufeatures 0.3.1", ] [[package]] @@ -265,6 +276,45 @@ dependencies = [ "libloading", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -303,6 +353,17 @@ dependencies = [ "slab", ] +[[package]] +name = "async-fs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" +dependencies = [ + "async-lock", + "blocking", + "futures-lite", +] + [[package]] name = "async-io" version = "2.6.0" @@ -421,11 +482,11 @@ dependencies = [ "enumflags2", "serde", "static_assertions", - "zbus", + "zbus 5.19.0", "zbus-lockstep", "zbus-lockstep-macros", - "zbus_names", - "zvariant", + "zbus_names 4.3.4", + "zvariant 5.15.0", ] [[package]] @@ -436,7 +497,7 @@ checksum = "2230e48787ed3eb4088996eab66a32ca20c0b67bbd4fd6cdfe79f04f1f04c9fc" dependencies = [ "atspi-common", "serde", - "zbus", + "zbus 5.19.0", ] [[package]] @@ -521,6 +582,9 @@ name = "bit-vec" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] [[package]] name = "bitflags" @@ -537,6 +601,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -546,6 +619,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "block-padding" version = "0.4.2" @@ -597,7 +679,7 @@ dependencies = [ "boa_macros", "boa_string", "indexmap", - "num-bigint", + "num-bigint 0.5.1", "rustc-hash 2.1.3", "strum", ] @@ -632,7 +714,7 @@ dependencies = [ "indexmap", "intrusive-collections", "itertools", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "num_enum", @@ -695,7 +777,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", ] [[package]] @@ -710,7 +792,7 @@ dependencies = [ "boa_macros", "fast-float2", "icu_properties", - "num-bigint", + "num-bigint 0.5.1", "num-traits", "regress", "rustc-hash 2.1.3", @@ -801,6 +883,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "byteorder-lite" version = "0.1.0" @@ -864,13 +952,22 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "cbc" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" dependencies = [ - "cipher", + "cipher 0.5.2", ] [[package]] @@ -913,7 +1010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -928,14 +1025,24 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout 0.1.4", +] + [[package]] name = "cipher" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ - "crypto-common", - "inout", + "crypto-common 0.2.2", + "inout 0.2.2", ] [[package]] @@ -1081,6 +1188,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -1156,6 +1272,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -1177,6 +1303,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix 0.31.3", + "windows-sys 0.61.2", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -1200,9 +1337,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "curve25519-dalek-derive", - "digest", + "digest 0.11.3", "fiat-crypto", "rustc_version", "subtle", @@ -1234,12 +1371,47 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "data-url" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" +dependencies = [ + "libc", + "libdbus-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "dbus-secret-service" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" +dependencies = [ + "aes 0.8.4", + "block-padding 0.3.3", + "cbc 0.1.2", + "dbus", + "fastrand", + "hkdf", + "num", + "once_cell", + "sha2 0.10.9", + "zeroize", +] + [[package]] name = "der" version = "0.8.2" @@ -1251,6 +1423,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint 0.4.8", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1263,7 +1449,18 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" dependencies = [ - "cipher", + "cipher 0.5.2", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -1272,9 +1469,9 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", - "crypto-common", + "crypto-common 0.2.2", "ctutils", ] @@ -1369,7 +1566,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26f2a8b3e564eba0877223dc343703ad0385794e882e6d13f3a4dd5c6b1f41ac" dependencies = [ - "cipher", + "cipher 0.5.2", ] [[package]] @@ -1379,7 +1576,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ "der", - "digest", + "digest 0.11.3", "elliptic-curve", "rfc6979", "signature", @@ -1415,7 +1612,7 @@ checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ "curve25519-dalek", "ed25519", - "sha2", + "sha2 0.11.0", "subtle", "zeroize", ] @@ -1582,8 +1779,8 @@ checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ "base16ct", "crypto-bigint", - "crypto-common", - "digest", + "crypto-common 0.2.2", + "digest 0.11.3", "ff", "group", "hybrid-array", @@ -2000,6 +2197,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + [[package]] name = "futures-task" version = "0.3.34" @@ -2013,12 +2216,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -2385,13 +2601,31 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -2585,13 +2819,23 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding 0.3.3", + "generic-array", +] + [[package]] name = "inout" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "block-padding", + "block-padding 0.4.2", "hybrid-array", ] @@ -2702,7 +2946,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", +] + +[[package]] +name = "keyring" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" +dependencies = [ + "byteorder", + "dbus-secret-service", + "log", + "secret-service", + "security-framework 2.11.1", + "security-framework 3.7.0", + "windows-sys 0.60.2", + "zbus 4.4.0", + "zeroize", ] [[package]] @@ -2755,12 +3016,27 @@ dependencies = [ "smallvec", ] +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "pkg-config", +] + [[package]] name = "libloading" version = "0.8.9" @@ -2838,10 +3114,10 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" name = "lopdf" version = "0.45.0" dependencies = [ - "aes", + "aes 0.9.3", "bitflags 2.13.2", "brotli-decompressor", - "cbc", + "cbc 0.2.1", "chrono", "ecb", "encoding_rs", @@ -2851,11 +3127,11 @@ dependencies = [ "itoa", "log", "md-5", - "nom", + "nom 8.0.0", "rand 0.10.3", "rangemap", "rayon", - "sha2", + "sha2 0.11.0", "stringprep", "thiserror 2.0.21", "weezl 0.2.1", @@ -2868,7 +3144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", - "digest", + "digest 0.11.3", ] [[package]] @@ -2895,6 +3171,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2999,12 +3281,47 @@ dependencies = [ "jni-sys 0.3.1", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nohash-hasher" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -3014,6 +3331,30 @@ dependencies = [ "memchr", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint 0.4.8", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.5.1" @@ -3025,6 +3366,15 @@ dependencies = [ "serde", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -3040,6 +3390,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3417,6 +3788,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3474,7 +3854,7 @@ dependencies = [ "elliptic-curve", "primefield", "primeorder", - "sha2", + "sha2 0.11.0", ] [[package]] @@ -3488,7 +3868,7 @@ dependencies = [ "fiat-crypto", "primefield", "primeorder", - "sha2", + "sha2 0.11.0", ] [[package]] @@ -3502,7 +3882,7 @@ dependencies = [ "elliptic-curve", "primefield", "primeorder", - "sha2", + "sha2 0.11.0", ] [[package]] @@ -3546,8 +3926,8 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ - "digest", - "hmac", + "digest 0.11.3", + "hmac 0.13.0", ] [[package]] @@ -3598,6 +3978,7 @@ dependencies = [ "image", "pdfcraft-cos", "pdfcraft-engine", + "pdfcraft-hot-folder-service", "pdfcraft-model", "pdfcraft-organize", "pdfcraft-platform", @@ -3605,6 +3986,8 @@ dependencies = [ "pdfcraft-xfa", "png 0.18.1", "serde_json", + "tempfile", + "zeroize", ] [[package]] @@ -3663,10 +4046,10 @@ dependencies = [ name = "pdfcraft-crypt" version = "0.6.0" dependencies = [ - "aes", + "aes 0.9.3", "md-5", "proptest", - "sha2", + "sha2 0.11.0", "stringprep", "thiserror 2.0.21", ] @@ -3693,9 +4076,11 @@ dependencies = [ "icu_properties", "idna", "image", + "keyring", "pdfcraft-a11y", "pdfcraft-annot", "pdfcraft-compare", + "pdfcraft-content", "pdfcraft-cos", "pdfcraft-create", "pdfcraft-edit", @@ -3716,8 +4101,19 @@ dependencies = [ "pdfcraft-xfa", "pdfcraft-xfdf", "png 0.18.1", + "rcgen", + "rustix 1.1.5", + "rustls", + "rustls-native-certs", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", "thiserror 2.0.21", + "ureq", "url", + "windows-permissions", + "zeroize", ] [[package]] @@ -3762,6 +4158,35 @@ dependencies = [ name = "pdfcraft-geom" version = "0.6.0" +[[package]] +name = "pdfcraft-hot-folder" +version = "0.6.0" +dependencies = [ + "chrono", + "eframe", + "egui", + "egui_kittest", + "pdfcraft-engine", + "pdfcraft-fonts", + "pdfcraft-hot-folder-service", + "rfd", + "rustix 1.1.5", + "serde_json", + "sys-locale", + "tempfile", + "zeroize", +] + +[[package]] +name = "pdfcraft-hot-folder-service" +version = "0.6.0" +dependencies = [ + "ctrlc", + "serde", + "tempfile", + "windows-service", +] + [[package]] name = "pdfcraft-js" version = "0.6.0" @@ -3828,7 +4253,7 @@ dependencies = [ name = "pdfcraft-platform" version = "0.6.0" dependencies = [ - "zbus", + "zbus 5.19.0", ] [[package]] @@ -3885,18 +4310,18 @@ dependencies = [ name = "pdfcraft-sign" version = "0.6.0" dependencies = [ - "aes", + "aes 0.9.3", "aws-lc-rs", "base64 0.22.1", "bp256", "bp384", - "cbc", + "cbc 0.2.1", "crypto-bigint", "des", "ecdsa", "ed25519-dalek", "getrandom 0.4.3", - "hmac", + "hmac 0.13.0", "p256", "p384", "p521", @@ -3908,9 +4333,9 @@ dependencies = [ "ripemd", "rsa", "rustls-cng", - "security-framework", - "sha1", - "sha2", + "security-framework 3.7.0", + "sha1 0.11.0", + "sha2 0.11.0", "sha3", "thiserror 2.0.21", ] @@ -3977,7 +4402,7 @@ dependencies = [ "pdfcraft-fonts", "png 0.18.1", "roxmltree", - "sha2", + "sha2 0.11.0", "thiserror 2.0.21", ] @@ -4304,7 +4729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ "crypto-bigint", - "crypto-common", + "crypto-common 0.2.2", "ff", "rand_core 0.10.1", "subtle", @@ -4359,7 +4784,7 @@ dependencies = [ "bitflags 2.13.2", "num-traits", "rand 0.9.5", - "rand_chacha", + "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -4415,13 +4840,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -4436,6 +4872,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -4446,6 +4892,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -4526,7 +4981,20 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ceda21af1ae61033b63175653a1af86cae399d79cd03ca80ba347eb3a6c4a7fe" dependencies = [ - "cipher", + "cipher 0.5.2", +] + +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", ] [[package]] @@ -4630,7 +5098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ "crypto-bigint", - "hmac", + "hmac 0.13.0", ] [[package]] @@ -4689,7 +5157,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd4211456b4172d7e44261920c25acf07367c4f04bb5f5d54fc21b090d9b159" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -4721,7 +5189,7 @@ dependencies = [ "const-oid", "crypto-bigint", "crypto-primes", - "digest", + "digest 0.11.3", "pkcs1", "pkcs8", "rand_core 0.10.1", @@ -4867,6 +5335,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "rustix" version = "0.38.44" @@ -4928,7 +5405,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.7.0", ] [[package]] @@ -5019,6 +5496,38 @@ dependencies = [ "zeroize", ] +[[package]] +name = "secret-service" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4d35ad99a181be0a60ffcbe85d680d98f87bdc4d7644ade319b87076b9dbfd4" +dependencies = [ + "aes 0.8.4", + "cbc 0.1.2", + "futures-util", + "generic-array", + "hkdf", + "num", + "once_cell", + "rand 0.8.8", + "serde", + "sha2 0.10.9", + "zbus 4.4.0", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -5127,6 +5636,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha1" version = "0.11.0" @@ -5134,8 +5654,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -5145,8 +5676,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -5155,7 +5686,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" dependencies = [ - "digest", + "digest 0.11.3", "keccak", "sponge-cursor", ] @@ -5182,7 +5713,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ - "digest", + "digest 0.11.3", "rand_core 0.10.1", ] @@ -5474,6 +6005,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -6473,6 +7015,28 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -6482,6 +7046,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" version = "0.62.2" @@ -6565,6 +7135,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-permissions" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2ccdc3c6bf4d4a094e031b63fadd08d8e42abd259940eb8aa5fdc09d4bf9be" +dependencies = [ + "bitflags 1.3.2", + "winapi", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -6574,6 +7154,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-service" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "857224b3b211c6f3616921f081ee54721ee3ad2ace2fac6a6337e032f7b4dcf2" +dependencies = [ + "bitflags 2.13.2", + "widestring", + "windows-sys 0.61.2", +] + [[package]] name = "windows-strings" version = "0.5.1" @@ -6893,12 +7484,40 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + [[package]] name = "xcursor" version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "163b33ed8786455e2fa5d72f554057ce3f3182425434f756cd39c99839d88e23" +[[package]] +name = "xdg-home" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "xkbcommon-dl" version = "0.4.2" @@ -6939,10 +7558,20 @@ dependencies = [ "png 0.18.1", "serde", "serde_json", - "sha2", + "sha2 0.11.0", "toml", ] +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.3" @@ -6963,7 +7592,45 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", +] + +[[package]] +name = "zbus" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" +dependencies = [ + "async-broadcast", + "async-executor", + "async-fs", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-sink", + "futures-util", + "hex", + "nix 0.29.0", + "ordered-stream", + "rand 0.8.8", + "serde", + "serde_repr", + "sha1 0.10.7", + "static_assertions", + "tracing", + "uds_windows", + "windows-sys 0.52.0", + "xdg-home", + "zbus_macros 4.4.0", + "zbus_names 3.0.0", + "zvariant 4.2.0", ] [[package]] @@ -6996,9 +7663,9 @@ dependencies = [ "uuid", "windows-sys 0.61.2", "winnow", - "zbus_macros", - "zbus_names", - "zvariant", + "zbus_macros 5.19.0", + "zbus_names 4.3.4", + "zvariant 5.15.0", ] [[package]] @@ -7008,7 +7675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6998de05217a084b7578728a9443d04ea4cd80f2a0839b8d78770b76ccd45863" dependencies = [ "zbus_xml", - "zvariant", + "zvariant 5.15.0", ] [[package]] @@ -7022,7 +7689,20 @@ dependencies = [ "syn 2.0.119", "zbus-lockstep", "zbus_xml", - "zvariant", + "zvariant 5.15.0", +] + +[[package]] +name = "zbus_macros" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils 2.1.0", ] [[package]] @@ -7035,9 +7715,20 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "zbus_names", - "zvariant", - "zvariant_utils", + "zbus_names 4.3.4", + "zvariant 5.15.0", + "zvariant_utils 4.2.0", +] + +[[package]] +name = "zbus_names" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" +dependencies = [ + "serde", + "static_assertions", + "zvariant 4.2.0", ] [[package]] @@ -7048,7 +7739,7 @@ checksum = "d8bf88b4a3ff53e883001e0e0115b297a9d53c31b9c1edd2bfdd853e3428624e" dependencies = [ "serde", "winnow", - "zvariant", + "zvariant 5.15.0", ] [[package]] @@ -7059,8 +7750,8 @@ checksum = "d1586c021a01ca0a9216dcd874e546382e156a5cbab5fab6cb5f10087e22682a" dependencies = [ "serde", "winnow", - "zbus_names", - "zvariant", + "zbus_names 4.3.4", + "zvariant 5.15.0", ] [[package]] @@ -7110,7 +7801,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", ] [[package]] @@ -7118,6 +7809,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" @@ -7179,6 +7884,19 @@ dependencies = [ "zune-core", ] +[[package]] +name = "zvariant" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" +dependencies = [ + "endi", + "enumflags2", + "serde", + "static_assertions", + "zvariant_derive 4.2.0", +] + [[package]] name = "zvariant" version = "5.15.0" @@ -7190,8 +7908,21 @@ dependencies = [ "serde", "winnow", "zcheapstr", - "zvariant_derive", - "zvariant_utils", + "zvariant_derive 5.15.0", + "zvariant_utils 4.2.0", +] + +[[package]] +name = "zvariant_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils 2.1.0", ] [[package]] @@ -7204,7 +7935,18 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "zvariant_utils", + "zvariant_utils 4.2.0", +] + +[[package]] +name = "zvariant_utils" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index 8d32b0dd98..6a6910acbf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -133,6 +133,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- 2026-10-11 — M13: optional standalone Hot Folder with durable local/network/FTP/Outlook/open-project processing, calendar schedules, safe source policies, OCR/scan preprocessing, analysis reports, grouped PDF/Office/image/EPUB outputs, headless tools and seven UI languages. Restricted-account boot services and optional Windows/macOS/Linux installers have lifecycle coverage; proprietary ABBYY formats remain incompatible; see [the scope inventory](docs/hot-folder.md). + - **2026-10-11 (M10, OCR languages and defaults):** Fold Dutch, Danish, Norwegian Bokmål, Swedish, Finnish and Portuguese into the scan/OCR work, with save/reopen regressions for accented words. The UI selects the primary system/browser language, independently of UI translations, with English fallback; manual selection remains available. Icelandic stays unclaimed. Overall percentage unchanged. - **2026-10-11 (M10, Italian OCR):** Added Italiano (it) to recognition in the UI, CLI and MCP. Both OCR schemas use the engine language list. A real-model regression checks all six Italian accented vowels after save/reopen and unchanged scan pixels; UI and single/batch automation cover selection. Overall percentage unchanged. diff --git a/apps/pdfcraft-hot-folder/Cargo.toml b/apps/pdfcraft-hot-folder/Cargo.toml new file mode 100644 index 0000000000..5a648a93e0 --- /dev/null +++ b/apps/pdfcraft-hot-folder/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "pdfcraft-hot-folder" +description = "Optional PdfCraft Hot Folder application and background worker" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lints] +workspace = true + +[dependencies] +pdfcraft-engine = { workspace = true } +pdfcraft-fonts = { workspace = true } +pdfcraft-hot-folder-service = { path = "../../crates/hot-folder-service" } +eframe = { workspace = true, features = ["default_fonts", "glow", "accesskit", "wayland", "x11"] } +egui = { workspace = true } +rfd = { workspace = true } +serde_json = { workspace = true } +sys-locale = "0.3" +chrono = { version = "0.4", default-features = false, features = ["clock"] } +tempfile = "3" +zeroize = "1.8" + +[target.'cfg(unix)'.dependencies] +rustix = { version = "1", features = ["process"] } + +[dev-dependencies] +egui_kittest = { workspace = true, features = ["eframe", "wgpu"] } diff --git a/apps/pdfcraft-hot-folder/src/i18n.rs b/apps/pdfcraft-hot-folder/src/i18n.rs new file mode 100644 index 0000000000..64ceebdf9c --- /dev/null +++ b/apps/pdfcraft-hot-folder/src/i18n.rs @@ -0,0 +1,1041 @@ +//! Complete small catalog for the standalone application. Engine diagnostics remain verbatim +//! under a translated error heading so paths and actionable technical details are preserved. + +pub const LANGUAGES: &[(&str, &str)] = + &[("en", "English"), ("de", "Deutsch"), ("fr", "Français"), ("nl", "Nederlands"), ("es", "Español"), ("nb", "Norsk bokmål"), ("sv", "Svenska")]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Language(pub usize); + +impl Language { + pub fn system(locale: Option<&str>) -> Self { + let code = pdfcraft_engine::ocr::language_for_locale(locale); + Self(LANGUAGES.iter().position(|l| l.0 == code).unwrap_or(0)) + } + pub fn t(self, key: &'static str) -> &'static str { + MESSAGES.iter().find(|row| row[0] == key).and_then(|row| row.get(self.0)).copied().unwrap_or(key) + } +} + +pub const MESSAGES: &[[&str; 7]] = &[ + [ + "Images (PNG, JPEG, TIFF)", + "Bilder (PNG, JPEG, TIFF)", + "Images (PNG, JPEG, TIFF)", + "Afbeeldingen (PNG, JPEG, TIFF)", + "Imágenes (PNG, JPEG, TIFF)", + "Bilder (PNG, JPEG, TIFF)", + "Bilder (PNG, JPEG, TIFF)", + ], + [ + "Status unavailable", + "Status nicht verfügbar", + "État indisponible", + "Status niet beschikbaar", + "Estado no disponible", + "Status utilgjengelig", + "Status saknas", + ], + [ + "Analyze without OCR", + "Ohne OCR analysieren", + "Analyser sans OCR", + "Analyseren zonder OCR", + "Analizar sin OCR", + "Analyser uten OCR", + "Analysera utan OCR", + ], + [ + "Layout report (JSON)", + "Layoutbericht (JSON)", + "Rapport de mise en page (JSON)", + "Lay-outrapport (JSON)", + "Informe de diseño (JSON)", + "Layout-rapport (JSON)", + "Layoutrapport (JSON)", + ], + [ + "Creates JSON layout reports. OCR, image changes and blank-page removal are disabled.", + "Erstellt JSON-Layoutberichte. OCR, Bildänderungen und das Entfernen leerer Seiten sind deaktiviert.", + "Crée des rapports JSON. L’OCR, les modifications d’image et la suppression des pages vides sont désactivés.", + "Maakt JSON-lay-outrapporten. OCR, beeldwijzigingen en het verwijderen van lege pagina’s zijn uitgeschakeld.", + "Crea informes JSON. Se desactivan el OCR, los cambios de imagen y la eliminación de páginas en blanco.", + "Oppretter JSON-rapporter. OCR, bildeendringer og fjerning av tomme sider er deaktivert.", + "Skapar JSON-rapporter. OCR, bildändringar och borttagning av tomma sidor är avstängda.", + ], + ["Excel (XLSX)", "Excel (XLSX)", "Excel (XLSX)", "Excel (XLSX)", "Excel (XLSX)", "Excel (XLSX)", "Excel (XLSX)"], + ["E-book (EPUB)", "E-Book (EPUB)", "Livre numérique (EPUB)", "E-book (EPUB)", "Libro electrónico (EPUB)", "E-bok (EPUB)", "E-bok (EPUB)"], + ["Source", "Quelle", "Source", "Bron", "Origen", "Kilde", "Källa"], + [ + "Staging folder", + "Zwischenablageordner", + "Dossier de transit", + "Tijdelijke invoermap", + "Carpeta de preparación", + "Mellomlagringsmappe", + "Mellanlagringsmapp", + ], + [ + "Local or network folder", + "Lokaler Ordner oder Netzwerkordner", + "Dossier local ou réseau", + "Lokale map of netwerkmap", + "Carpeta local o de red", + "Lokal mappe eller nettverksmappe", + "Lokal mapp eller nätverksmapp", + ], + ["FTP / FTPS", "FTP / FTPS", "FTP / FTPS", "FTP / FTPS", "FTP / FTPS", "FTP / FTPS", "FTP / FTPS"], + [ + "Outlook attachments", + "Outlook-Anhänge", + "Pièces jointes Outlook", + "Outlook-bijlagen", + "Adjuntos de Outlook", + "Outlook-vedlegg", + "Outlook-bilagor", + ], + ["PDFCraft project", "PDFCraft-Projekt", "Projet PDFCraft", "PDFCraft-project", "Proyecto PDFCraft", "PDFCraft-prosjekt", "PDFCraft-projekt"], + ["Server", "Server", "Serveur", "Server", "Servidor", "Server", "Server"], + ["Port", "Port", "Port", "Poort", "Puerto", "Port", "Port"], + ["FTP with TLS", "FTP mit TLS", "FTP avec TLS", "FTP met TLS", "FTP con TLS", "FTP med TLS", "FTP med TLS"], + ["Unencrypted FTP", "Unverschlüsseltes FTP", "FTP non chiffré", "Onversleutelde FTP", "FTP sin cifrar", "Ukryptert FTP", "Okrypterad FTP"], + ["Username", "Benutzername", "Nom d’utilisateur", "Gebruikersnaam", "Nombre de usuario", "Brukernavn", "Användarnamn"], + ["Server folder", "Serverordner", "Dossier du serveur", "Servermap", "Carpeta del servidor", "Servermappe", "Servermapp"], + [ + "Credential ID", + "Zugangsdaten-ID", + "Identifiant des données d’accès", + "ID voor aanmeldgegevens", + "ID de credenciales", + "ID for påloggingsopplysninger", + "ID för inloggningsuppgifter", + ], + ["Password", "Passwort", "Mot de passe", "Wachtwoord", "Contraseña", "Passord", "Lösenord"], + [ + "Save password securely", + "Passwort sicher speichern", + "Enregistrer le mot de passe en sécurité", + "Wachtwoord veilig opslaan", + "Guardar contraseña de forma segura", + "Lagre passord sikkert", + "Spara lösenord säkert", + ], + [ + "Archive server folder", + "Archivordner auf dem Server", + "Dossier d’archivage du serveur", + "Archiefmap op server", + "Carpeta de archivo del servidor", + "Arkivmappe på serveren", + "Arkivmapp på servern", + ], + [ + "Application ID (public client)", + "Anwendungs-ID (öffentlicher Client)", + "ID d’application (client public)", + "Toepassings-ID (openbare client)", + "ID de aplicación (cliente público)", + "Program-ID (offentlig klient)", + "Program-ID (offentlig klient)", + ], + ["Tenant", "Mandant", "Locataire", "Tenant", "Inquilino", "Leier", "Klientorganisation"], + [ + "Use your own Microsoft application registration with public-client device sign-in enabled.", + "Eigene Microsoft-App-Registrierung mit aktivierter Geräteanmeldung für öffentliche Clients verwenden.", + "Utilisez votre propre application Microsoft avec la connexion par appareil activée pour les clients publics.", + "Gebruik een eigen Microsoft-appregistratie met apparaataanmelding voor openbare clients.", + "Use su propia aplicación de Microsoft con inicio por dispositivo para clientes públicos habilitado.", + "Bruk din egen Microsoft-appregistrering med enhetspålogging for offentlige klienter aktivert.", + "Använd en egen Microsoft-appregistrering med enhetsinloggning för offentliga klienter aktiverad.", + ], + [ + "Sign in to Outlook", + "Bei Outlook anmelden", + "Se connecter à Outlook", + "Aanmelden bij Outlook", + "Iniciar sesión en Outlook", + "Logg på Outlook", + "Logga in på Outlook", + ], + [ + "Load mail folders", + "E-Mail-Ordner laden", + "Charger les dossiers de messagerie", + "E-mailmappen laden", + "Cargar carpetas de correo", + "Last inn e-postmapper", + "Läs in e-postmappar", + ], + [ + "Mail folder ID", + "E-Mail-Ordner-ID", + "ID du dossier de messagerie", + "ID van e-mailmap", + "ID de carpeta de correo", + "ID for e-postmappe", + "ID för e-postmapp", + ], + [ + "Open subfolders", + "Unterordner öffnen", + "Ouvrir les sous-dossiers", + "Submappen openen", + "Abrir subcarpetas", + "Åpne undermapper", + "Öppna undermappar", + ], + [ + "Archive mail folder ID", + "ID des E-Mail-Archivordners", + "ID du dossier d’archivage des messages", + "ID van e-mailarchiefmap", + "ID de carpeta de archivo de correo", + "ID for e-postarkivmappe", + "ID för e-postarkivmapp", + ], + [ + "Mail move/delete affects the whole message and waits for all supported attachments. Enable PDF and images.", + "Verschieben/Löschen betrifft die ganze Nachricht und wartet auf alle unterstützten Anhänge. PDF und Bilder aktivieren.", + "Déplacer/supprimer concerne le message entier et attend toutes les pièces jointes prises en charge. Activez PDF et images.", + "Verplaatsen/verwijderen betreft het hele bericht en wacht op alle ondersteunde bijlagen. Schakel PDF en afbeeldingen in.", + "Mover/eliminar afecta al mensaje completo y espera todos los adjuntos compatibles. Active PDF e imágenes.", + "Flytting/sletting gjelder hele meldingen og venter på alle støttede vedlegg. Aktiver PDF og bilder.", + "Flytt/radering gäller hela meddelandet och väntar på alla bilagor som stöds. Aktivera PDF och bilder.", + ], + [ + "Project manifest", + "Projektmanifest", + "Manifeste du projet", + "Projectmanifest", + "Manifiesto del proyecto", + "Prosjektmanifest", + "Projektmanifest", + ], + ["Choose project…", "Projekt auswählen…", "Choisir un projet…", "Project kiezen…", "Elegir proyecto…", "Velg prosjekt…", "Välj projekt…"], + [ + "Create project from files…", + "Projekt aus Dateien erstellen…", + "Créer un projet à partir de fichiers…", + "Project uit bestanden maken…", + "Crear proyecto a partir de archivos…", + "Opprett prosjekt fra filer…", + "Skapa projekt från filer…", + ], + [ + "Open PDFCraft manifests contain an ordered list of PDF/image files. Proprietary ABBYY projects are not supported.", + "Offene PDFCraft-Manifeste enthalten eine geordnete Liste von PDF-/Bilddateien. Proprietäre ABBYY-Projekte werden nicht unterstützt.", + "Les manifestes ouverts PDFCraft contiennent une liste ordonnée de PDF/images. Les projets propriétaires ABBYY ne sont pas pris en charge.", + "Open PDFCraft-manifesten bevatten een geordende lijst PDF-/afbeeldingsbestanden. Eigen ABBYY-projecten worden niet ondersteund.", + "Los manifiestos abiertos de PDFCraft contienen una lista ordenada de PDF/imágenes. No se admiten proyectos propietarios de ABBYY.", + "Åpne PDFCraft-manifester inneholder en ordnet liste med PDF-/bildefiler. Proprietære ABBYY-prosjekter støttes ikke.", + "Öppna PDFCraft-manifest innehåller en ordnad lista med PDF-/bildfiler. Proprietära ABBYY-projekt stöds inte.", + ], + ["Connecting…", "Verbindung wird hergestellt…", "Connexion…", "Verbinden…", "Conectando…", "Kobler til…", "Ansluter…"], + [ + "Open the sign-in page and enter this code:", + "Anmeldeseite öffnen und diesen Code eingeben:", + "Ouvrez la page de connexion et saisissez ce code :", + "Open de aanmeldpagina en voer deze code in:", + "Abra la página de inicio de sesión e introduzca este código:", + "Åpne påloggingssiden og skriv inn denne koden:", + "Öppna inloggningssidan och ange denna kod:", + ], + [ + "Check credentials", + "Zugangsdaten prüfen", + "Vérifier les identifiants", + "Aanmeldgegevens controleren", + "Comprobar credenciales", + "Kontroller påloggingsopplysninger", + "Kontrollera inloggningsuppgifter", + ], + [ + "Remove credentials", + "Zugangsdaten entfernen", + "Supprimer les identifiants", + "Aanmeldgegevens verwijderen", + "Eliminar credenciales", + "Fjern påloggingsopplysninger", + "Ta bort inloggningsuppgifter", + ], + [ + "Credentials stay in the protected service store and are never exported with tasks.", + "Zugangsdaten bleiben im geschützten Dienstspeicher und werden nie mit Aufgaben exportiert.", + "Les identifiants restent dans le stockage protégé du service et ne sont jamais exportés avec les tâches.", + "Aanmeldgegevens blijven in de beveiligde dienstopslag en worden nooit met taken geëxporteerd.", + "Las credenciales permanecen en el almacén protegido del servicio y nunca se exportan con las tareas.", + "Påloggingsopplysninger blir i det beskyttede tjenestelageret og eksporteres aldri med oppgaver.", + "Inloggningsuppgifter finns kvar i tjänstens skyddade lagring och exporteras aldrig med uppgifter.", + ], + [ + "After processing originals", + "Originale nach der Verarbeitung", + "Originaux après traitement", + "Originelen na verwerking", + "Originales después del procesamiento", + "Originaler etter behandling", + "Original efter bearbetning", + ], + [ + "Credentials saved", + "Zugangsdaten gespeichert", + "Identifiants enregistrés", + "Aanmeldgegevens opgeslagen", + "Credenciales guardadas", + "Påloggingsopplysninger lagret", + "Inloggningsuppgifter sparade", + ], + [ + "Credentials removed", + "Zugangsdaten entfernt", + "Identifiants supprimés", + "Aanmeldgegevens verwijderd", + "Credenciales eliminadas", + "Påloggingsopplysninger fjernet", + "Inloggningsuppgifter borttagna", + ], + [ + "Credentials available", + "Zugangsdaten vorhanden", + "Identifiants disponibles", + "Aanmeldgegevens beschikbaar", + "Credenciales disponibles", + "Påloggingsopplysninger tilgjengelige", + "Inloggningsuppgifter tillgängliga", + ], + [ + "Sign-in required", + "Anmeldung erforderlich", + "Connexion requise", + "Aanmelding vereist", + "Se requiere iniciar sesión", + "Pålogging kreves", + "Inloggning krävs", + ], + ["Signed in", "Angemeldet", "Connecté", "Aangemeld", "Sesión iniciada", "Pålogget", "Inloggad"], + [ + "Sign-in expired", + "Anmeldung abgelaufen", + "Connexion expirée", + "Aanmelding verlopen", + "Inicio de sesión caducado", + "Pålogging utløpt", + "Inloggningen har gått ut", + ], + [ + "Sign-in declined", + "Anmeldung abgelehnt", + "Connexion refusée", + "Aanmelding geweigerd", + "Inicio de sesión rechazado", + "Pålogging avvist", + "Inloggning nekad", + ], + [ + "Unsaved changes", + "Ungespeicherte Änderungen", + "Modifications non enregistrées", + "Niet-opgeslagen wijzigingen", + "Cambios sin guardar", + "Ulagrede endringer", + "Osparade ändringar", + ], + [ + "Settings changed in another window. Reload before saving.", + "Die Einstellungen wurden in einem anderen Fenster geändert. Vor dem Speichern neu laden.", + "Les paramètres ont changé dans une autre fenêtre. Rechargez avant d’enregistrer.", + "Instellingen zijn in een ander venster gewijzigd. Herlaad voor het opslaan.", + "La configuración cambió en otra ventana. Recargue antes de guardar.", + "Innstillingene ble endret i et annet vindu. Last inn på nytt før lagring.", + "Inställningarna ändrades i ett annat fönster. Läs in igen innan du sparar.", + ], + [ + "Discard edits and reload", + "Änderungen verwerfen und neu laden", + "Abandonner et recharger", + "Wijzigingen verwerpen en herladen", + "Descartar cambios y recargar", + "Forkast endringer og last inn igjen", + "Kasta ändringar och läs in igen", + ], + [ + "Save your task settings before closing?", + "Aufgabeneinstellungen vor dem Schließen speichern?", + "Enregistrer les paramètres avant de fermer ?", + "Taakinstellingen opslaan voor het sluiten?", + "¿Guardar la configuración antes de cerrar?", + "Lagre oppgaveinnstillingene før lukking?", + "Spara uppgiftsinställningarna innan du stänger?", + ], + [ + "Save and close", + "Speichern und schließen", + "Enregistrer et fermer", + "Opslaan en sluiten", + "Guardar y cerrar", + "Lagre og lukk", + "Spara och stäng", + ], + [ + "Discard and close", + "Verwerfen und schließen", + "Abandonner et fermer", + "Verwerpen en sluiten", + "Descartar y cerrar", + "Forkast og lukk", + "Kasta och stäng", + ], + ["Cancel", "Abbrechen", "Annuler", "Annuleren", "Cancelar", "Avbryt", "Avbryt"], + [ + "To edit protected service settings, reopen Hot Folder as administrator with the same configuration path.", + "Zum Bearbeiten geschützter Diensteinstellungen Hot Folder mit demselben Konfigurationspfad als Administrator öffnen.", + "Pour modifier les paramètres protégés, rouvrez Hot Folder en administrateur avec le même chemin de configuration.", + "Open Hot Folder als beheerder met hetzelfde configuratiepad om beschermde dienstinstellingen te wijzigen.", + "Para editar la configuración protegida, abra Hot Folder como administrador con la misma ruta de configuración.", + "Åpne Hot Folder som administrator med samme konfigurasjonssti for å redigere beskyttede tjenesteinnstillinger.", + "Öppna Hot Folder som administratör med samma konfigurationssökväg för att redigera skyddade tjänsteinställningar.", + ], + [ + "Page selection and image preparation", + "Seitenauswahl und Bildvorbereitung", + "Sélection des pages et préparation des images", + "Paginaselectie en beeldvoorbereiding", + "Selección de páginas y preparación de imágenes", + "Sidevalg og bildeforberedelse", + "Sidval och bildförberedelse", + ], + [ + "Pages to process (empty = all; example: 1-3,5)", + "Zu verarbeitende Seiten (leer = alle; Beispiel: 1-3,5)", + "Pages à traiter (vide = toutes ; exemple : 1-3,5)", + "Te verwerken pagina’s (leeg = alle; voorbeeld: 1-3,5)", + "Páginas a procesar (vacío = todas; ejemplo: 1-3,5)", + "Sider å behandle (tomt = alle; eksempel: 1-3,5)", + "Sidor att bearbeta (tomt = alla; exempel: 1-3,5)", + ], + [ + "Only selected pages are included in the output.", + "Nur ausgewählte Seiten werden ausgegeben.", + "Seules les pages sélectionnées sont incluses dans la sortie.", + "Alleen geselecteerde pagina’s worden uitgevoerd.", + "La salida incluye solo las páginas seleccionadas.", + "Bare valgte sider tas med i utdataene.", + "Endast valda sidor ingår i utdata.", + ], + [ + "Convert to grayscale", + "In Graustufen umwandeln", + "Convertir en niveaux de gris", + "Omzetten naar grijstinten", + "Convertir a escala de grises", + "Konverter til gråtoner", + "Konvertera till gråskala", + ], + ["Invert colors", "Farben umkehren", "Inverser les couleurs", "Kleuren omkeren", "Invertir colores", "Inverter farger", "Invertera färger"], + [ + "Reduce noise (median filter)", + "Rauschen reduzieren (Medianfilter)", + "Réduire le bruit (filtre médian)", + "Ruis verminderen (mediaanfilter)", + "Reducir ruido (filtro de mediana)", + "Reduser støy (medianfilter)", + "Minska brus (medianfilter)", + ], + [ + "Split spreads into left and right pages", + "Doppelseiten in linke und rechte Seite teilen", + "Séparer les doubles pages en pages gauche et droite", + "Dubbele pagina’s splitsen in links en rechts", + "Dividir páginas dobles en izquierda y derecha", + "Del oppslag i venstre og høyre side", + "Dela uppslag i vänster och höger sida", + ], + ["Brightness", "Helligkeit", "Luminosité", "Helderheid", "Brillo", "Lysstyrke", "Ljusstyrka"], + ["Contrast", "Kontrast", "Contraste", "Contrast", "Contraste", "Kontrast", "Kontrast"], + [ + "Convert to black and white", + "In Schwarzweiß umwandeln", + "Convertir en noir et blanc", + "Omzetten naar zwart-wit", + "Convertir a blanco y negro", + "Konverter til svart-hvitt", + "Konvertera till svartvitt", + ], + ["Threshold", "Schwellwert", "Seuil", "Drempel", "Umbral", "Terskel", "Tröskelvärde"], + [ + "Image changes rasterize pages at the chosen resolution. Documents with forms, links or annotations are refused.", + "Bildänderungen rastern Seiten in der gewählten Auflösung. Dokumente mit Formularen, Links oder Anmerkungen werden abgelehnt.", + "Les modifications d’image pixellisent les pages à la résolution choisie. Les documents avec formulaires, liens ou annotations sont refusés.", + "Beeldwijzigingen rasteren pagina’s op de gekozen resolutie. Documenten met formulieren, links of annotaties worden geweigerd.", + "Los cambios de imagen rasterizan las páginas a la resolución elegida. Se rechazan documentos con formularios, enlaces o anotaciones.", + "Bildeendringer rasteriserer sider ved valgt oppløsning. Dokumenter med skjemaer, lenker eller merknader avvises.", + "Bildändringar rastrerar sidor med vald upplösning. Dokument med formulär, länkar eller kommentarer avvisas.", + ], + [ + "Recognition areas (saved with task)", + "Erkennungsbereiche (in Aufgabe gespeichert)", + "Zones de reconnaissance (enregistrées avec la tâche)", + "Herkenningsgebieden (opgeslagen bij de taak)", + "Áreas de reconocimiento (guardadas con la tarea)", + "Gjenkjenningsområder (lagres med oppgaven)", + "Igenkänningsområden (sparas med uppgiften)", + ], + [ + "Coordinates are percentages from the top left. Page 0 applies to every prepared page. Visible content remains unchanged.", + "Koordinaten in Prozent ab links oben. Seite 0 gilt für jede vorbereitete Seite. Sichtbare Inhalte bleiben erhalten.", + "Coordonnées en pourcentage depuis le coin supérieur gauche. La page 0 s’applique à chaque page préparée. Le contenu visible reste inchangé.", + "Coördinaten zijn percentages vanaf linksboven. Pagina 0 geldt voor elke voorbereide pagina. Zichtbare inhoud blijft behouden.", + "Coordenadas en porcentajes desde arriba a la izquierda. La página 0 se aplica a todas las páginas preparadas. Se conserva el contenido visible.", + "Koordinater i prosent fra øvre venstre hjørne. Side 0 gjelder alle forberedte sider. Synlig innhold beholdes.", + "Koordinater i procent från övre vänstra hörnet. Sida 0 gäller alla förberedda sidor. Synligt innehåll bevaras.", + ], + ["Page", "Seite", "Page", "Pagina", "Página", "Side", "Sida"], + [ + "Exclude this area", + "Diesen Bereich ausschließen", + "Exclure cette zone", + "Dit gebied uitsluiten", + "Excluir esta área", + "Utelat dette området", + "Uteslut detta område", + ], + ["Remove area", "Bereich entfernen", "Supprimer la zone", "Gebied verwijderen", "Eliminar área", "Fjern område", "Ta bort område"], + [ + "Add recognition area", + "Erkennungsbereich hinzufügen", + "Ajouter une zone", + "Herkenningsgebied toevoegen", + "Añadir área de reconocimiento", + "Legg til gjenkjenningsområde", + "Lägg till igenkänningsområde", + ], + ["Left (%)", "Links (%)", "Gauche (%)", "Links (%)", "Izquierda (%)", "Venstre (%)", "Vänster (%)"], + ["Top (%)", "Oben (%)", "Haut (%)", "Boven (%)", "Arriba (%)", "Topp (%)", "Överkant (%)"], + ["Width (%)", "Breite (%)", "Largeur (%)", "Breedte (%)", "Ancho (%)", "Bredde (%)", "Bredd (%)"], + ["Height (%)", "Höhe (%)", "Hauteur (%)", "Hoogte (%)", "Alto (%)", "Høyde (%)", "Höjd (%)"], + [ + "Save removed blank pages separately", + "Entfernte Leerseiten separat speichern", + "Enregistrer séparément les pages blanches supprimées", + "Verwijderde lege pagina’s apart opslaan", + "Guardar aparte las páginas vacías eliminadas", + "Lagre fjernede blanke sider separat", + "Spara borttagna tomma sidor separat", + ], + [ + "Blank page archive", + "Ablage für Leerseiten", + "Archive des pages blanches", + "Archief voor lege pagina’s", + "Archivo de páginas vacías", + "Arkiv for blanke sider", + "Arkiv för tomma sidor", + ], + ["System service", "Systemdienst", "Service système", "Systeemdienst", "Servicio del sistema", "Systemtjeneste", "Systemtjänst"], + ["Running", "Läuft", "En cours", "Actief", "En ejecución", "Kjører", "Körs"], + ["Not installed", "Nicht installiert", "Non installé", "Niet geïnstalleerd", "No instalado", "Ikke installert", "Inte installerad"], + ["Service account", "Dienstkonto", "Compte du service", "Dienstaccount", "Cuenta del servicio", "Tjenestekonto", "Tjänstekonto"], + [ + "Start with the system", + "Beim Systemstart starten", + "Démarrer avec le système", + "Starten bij systeemstart", + "Iniciar con el sistema", + "Start ved systemoppstart", + "Starta vid systemstart", + ], + ["Start service", "Dienst starten", "Démarrer le service", "Dienst starten", "Iniciar servicio", "Start tjenesten", "Starta tjänsten"], + ["Stop service", "Dienst anhalten", "Arrêter le service", "Dienst stoppen", "Detener servicio", "Stopp tjenesten", "Stoppa tjänsten"], + ["Refresh status", "Status aktualisieren", "Actualiser l’état", "Status vernieuwen", "Actualizar estado", "Oppdater status", "Uppdatera status"], + [ + "Install service", + "Dienst installieren", + "Installer le service", + "Dienst installeren", + "Instalar servicio", + "Installer tjenesten", + "Installera tjänsten", + ], + [ + "Uninstall service", + "Dienst entfernen", + "Désinstaller le service", + "Dienst verwijderen", + "Desinstalar servicio", + "Avinstaller tjenesten", + "Avinstallera tjänsten", + ], + [ + "Install or remove the system service with the optional Windows installer component.", + "Installieren oder entfernen Sie den Dienst über die optionale Komponente im Windows-Installer.", + "Installez ou supprimez le service avec le composant facultatif de l’installeur Windows.", + "Installeer of verwijder de dienst via het optionele onderdeel van het Windows-installatieprogramma.", + "Instale o elimine el servicio mediante el componente opcional del instalador de Windows.", + "Installer eller fjern tjenesten med den valgfrie komponenten i Windows-installasjonsprogrammet.", + "Installera eller ta bort tjänsten med den valfria komponenten i Windows-installationsprogrammet.", + ], + [ + "Service changes require administrator rights. Grant the service account access to the selected folders.", + "Dienständerungen benötigen Administratorrechte. Gewähren Sie dem Dienstkonto Zugriff auf die gewählten Ordner.", + "Les modifications du service nécessitent des droits administrateur. Autorisez le compte du service à accéder aux dossiers choisis.", + "Dienstwijzigingen vereisen beheerdersrechten. Geef het dienstaccount toegang tot de gekozen mappen.", + "Los cambios del servicio requieren permisos de administrador. Conceda a la cuenta acceso a las carpetas seleccionadas.", + "Tjenesteendringer krever administratorrettigheter. Gi tjenestekontoen tilgang til de valgte mappene.", + "Tjänsteändringar kräver administratörsrättigheter. Ge tjänstekontot åtkomst till de valda mapparna.", + ], + ["Output saving steps", "Ausgabeschritte", "Étapes d’enregistrement", "Opslagstappen", "Pasos de guardado", "Lagringstrinn", "Sparsteg"], + [ + "All outputs must be saved before originals are moved or deleted. Existing files are never overwritten.", + "Erst nach dem Speichern aller Ausgaben werden Originale verschoben oder gelöscht. Vorhandene Dateien werden nie überschrieben.", + "Toutes les sorties doivent être enregistrées avant de déplacer ou supprimer les originaux. Les fichiers existants ne sont jamais écrasés.", + "Alle uitvoer wordt opgeslagen voordat originelen worden verplaatst of verwijderd. Bestaande bestanden worden nooit overschreven.", + "Todas las salidas se guardan antes de mover o eliminar originales. Nunca se sobrescriben archivos existentes.", + "Alle utdata lagres før originaler flyttes eller slettes. Eksisterende filer overskrives aldri.", + "Alla utdata sparas innan original flyttas eller raderas. Befintliga filer skrivs aldrig över.", + ], + ["Remove step", "Schritt entfernen", "Supprimer l’étape", "Stap verwijderen", "Eliminar paso", "Fjern trinn", "Ta bort steg"], + [ + "Add saving step", + "Ausgabeschritt hinzufügen", + "Ajouter une étape", + "Opslagstap toevoegen", + "Añadir paso de guardado", + "Legg til lagringstrinn", + "Lägg till sparsteg", + ], + [ + "One document per file", + "Ein Dokument je Datei", + "Un document par fichier", + "Eén document per bestand", + "Un documento por archivo", + "Ett dokument per fil", + "Ett dokument per fil", + ], + [ + "Combine files in each folder", + "Dateien je Ordner zusammenführen", + "Fusionner les fichiers de chaque dossier", + "Bestanden per map samenvoegen", + "Combinar archivos de cada carpeta", + "Slå sammen filer i hver mappe", + "Slå samman filer i varje mapp", + ], + [ + "Combine all files in this run", + "Alle Dateien dieses Durchlaufs zusammenführen", + "Fusionner tous les fichiers de cette exécution", + "Alle bestanden van deze uitvoering samenvoegen", + "Combinar todos los archivos de esta ejecución", + "Slå sammen alle filer i denne kjøringen", + "Slå samman alla filer i denna körning", + ], + [ + "Use another output folder", + "Anderen Ausgabeordner verwenden", + "Utiliser un autre dossier de sortie", + "Andere uitvoermap gebruiken", + "Usar otra carpeta de salida", + "Bruk en annen utdatamappe", + "Använd en annan utdatamapp", + ], + [ + "Filename template (without extension)", + "Dateinamenmuster (ohne Erweiterung)", + "Modèle de nom de fichier (sans extension)", + "Bestandsnaamsjabloon (zonder extensie)", + "Plantilla de nombre (sin extensión)", + "Filnavnmal (uten filtype)", + "Filnamnsmall (utan filändelse)", + ], + ["JPEG quality", "JPEG-Qualität", "Qualité JPEG", "JPEG-kwaliteit", "Calidad JPEG", "JPEG-kvalitet", "JPEG-kvalitet"], + [ + "PDF (processed document)", + "PDF (verarbeitetes Dokument)", + "PDF (document traité)", + "PDF (verwerkt document)", + "PDF (documento procesado)", + "PDF (behandlet dokument)", + "PDF (bearbetat dokument)", + ], + [ + "PDF (image only)", + "PDF (nur Bilder)", + "PDF (images seules)", + "PDF (alleen afbeeldingen)", + "PDF (solo imágenes)", + "PDF (bare bilder)", + "PDF (endast bilder)", + ], + ["Word (DOCX)", "Word (DOCX)", "Word (DOCX)", "Word (DOCX)", "Word (DOCX)", "Word (DOCX)", "Word (DOCX)"], + ["HTML", "HTML", "HTML", "HTML", "HTML", "HTML", "HTML"], + [ + "Rich text (RTF)", + "Rich Text (RTF)", + "Texte enrichi (RTF)", + "Opgemaakte tekst (RTF)", + "Texto enriquecido (RTF)", + "Rik tekst (RTF)", + "Formaterad text (RTF)", + ], + [ + "Plain text (TXT)", + "Nur Text (TXT)", + "Texte brut (TXT)", + "Platte tekst (TXT)", + "Texto sin formato (TXT)", + "Ren tekst (TXT)", + "Oformaterad text (TXT)", + ], + ["PNG images", "PNG-Bilder", "Images PNG", "PNG-afbeeldingen", "Imágenes PNG", "PNG-bilder", "PNG-bilder"], + ["JPEG images", "JPEG-Bilder", "Images JPEG", "JPEG-afbeeldingen", "Imágenes JPEG", "JPEG-bilder", "JPEG-bilder"], + ["TIFF images", "TIFF-Bilder", "Images TIFF", "TIFF-afbeeldingen", "Imágenes TIFF", "TIFF-bilder", "TIFF-bilder"], + [ + "At regular intervals", + "In regelmäßigen Abständen", + "À intervalles réguliers", + "Met regelmatige tussenpozen", + "A intervalos regulares", + "Med jevne mellomrom", + "Med jämna mellanrum", + ], + ["Daily", "Täglich", "Chaque jour", "Dagelijks", "Diariamente", "Daglig", "Dagligen"], + ["Weekly", "Wöchentlich", "Chaque semaine", "Wekelijks", "Semanalmente", "Ukentlig", "Varje vecka"], + ["Monthly", "Monatlich", "Chaque mois", "Maandelijks", "Mensualmente", "Månedlig", "Varje månad"], + ["Day of month", "Tag im Monat", "Jour du mois", "Dag van de maand", "Día del mes", "Dag i måneden", "Dag i månaden"], + ["Time", "Uhrzeit", "Heure", "Tijd", "Hora", "Tid", "Tid"], + ["Monday", "Montag", "Lundi", "Maandag", "Lunes", "Mandag", "Måndag"], + ["Tuesday", "Dienstag", "Mardi", "Dinsdag", "Martes", "Tirsdag", "Tisdag"], + ["Wednesday", "Mittwoch", "Mercredi", "Woensdag", "Miércoles", "Onsdag", "Onsdag"], + ["Thursday", "Donnerstag", "Jeudi", "Donderdag", "Jueves", "Torsdag", "Torsdag"], + ["Friday", "Freitag", "Vendredi", "Vrijdag", "Viernes", "Fredag", "Fredag"], + ["Saturday", "Samstag", "Samedi", "Zaterdag", "Sábado", "Lørdag", "Lördag"], + ["Sunday", "Sonntag", "Dimanche", "Zondag", "Domingo", "Søndag", "Söndag"], + [ + "Import tasks…", + "Aufgaben importieren…", + "Importer des tâches…", + "Taken importeren…", + "Importar tareas…", + "Importer oppgaver…", + "Importera uppgifter…", + ], + [ + "Export tasks…", + "Aufgaben exportieren…", + "Exporter les tâches…", + "Taken exporteren…", + "Exportar tareas…", + "Eksporter oppgaver…", + "Exportera uppgifter…", + ], + ["Start now", "Jetzt starten", "Démarrer maintenant", "Nu starten", "Iniciar ahora", "Start nå", "Starta nu"], + [ + "Start on schedule", + "Nach Zeitplan starten", + "Démarrer selon le planning", + "Starten volgens planning", + "Iniciar según programación", + "Start etter tidsplan", + "Starta enligt schema", + ], + ["Next check", "Nächste Prüfung", "Prochaine vérification", "Volgende controle", "Próxima comprobación", "Neste kontroll", "Nästa kontroll"], + [ + "Show errors only", + "Nur Fehler anzeigen", + "Afficher seulement les erreurs", + "Alleen fouten weergeven", + "Mostrar solo errores", + "Vis bare feil", + "Visa endast fel", + ], + ["Log", "Protokoll", "Journal", "Logboek", "Registro", "Logg", "Logg"], + [ + "Open output folder", + "Ausgabeordner öffnen", + "Ouvrir le dossier de sortie", + "Uitvoermap openen", + "Abrir carpeta de salida", + "Åpne utdatamappe", + "Öppna utdatamapp", + ], + [ + "Remove completely white pages", + "Vollständig weiße Seiten entfernen", + "Supprimer les pages entièrement blanches", + "Volledig witte pagina’s verwijderen", + "Eliminar páginas completamente blancas", + "Fjern helt hvite sider", + "Ta bort helt vita sidor", + ], + [ + "Faint marks and uncertain blank pages are preserved.", + "Schwache Markierungen und nicht eindeutig leere Seiten bleiben erhalten.", + "Les marques pâles et pages vides incertaines sont conservées.", + "Vage markeringen en onzekere lege pagina’s blijven behouden.", + "Se conservan las marcas tenues y páginas vacías dudosas.", + "Svake merker og usikre blanke sider beholdes.", + "Svaga markeringar och osäkra tomma sidor bevaras.", + ], + ["Tasks", "Aufgaben", "Tâches", "Taken", "Tareas", "Oppgaver", "Uppgifter"], + ["New task", "Neue Aufgabe", "Nouvelle tâche", "Nieuwe taak", "Nueva tarea", "Ny oppgave", "Ny uppgift"], + ["Duplicate", "Duplizieren", "Dupliquer", "Dupliceren", "Duplicar", "Dupliser", "Duplicera"], + ["Delete task", "Aufgabe löschen", "Supprimer la tâche", "Taak verwijderen", "Eliminar tarea", "Slett oppgave", "Ta bort uppgift"], + ["Save", "Speichern", "Enregistrer", "Opslaan", "Guardar", "Lagre", "Spara"], + ["Start", "Starten", "Démarrer", "Starten", "Iniciar", "Start", "Starta"], + ["Stop", "Stoppen", "Arrêter", "Stoppen", "Detener", "Stopp", "Stoppa"], + ["Stop all", "Alle stoppen", "Tout arrêter", "Alles stoppen", "Detener todas", "Stopp alle", "Stoppa alla"], + [ + "Retry failed files", + "Fehlgeschlagene Dateien erneut versuchen", + "Réessayer les fichiers en échec", + "Mislukte bestanden opnieuw proberen", + "Reintentar archivos fallidos", + "Prøv mislykkede filer på nytt", + "Försök igen med misslyckade filer", + ], + ["Task name", "Aufgabenname", "Nom de la tâche", "Taaknaam", "Nombre de la tarea", "Oppgavenavn", "Uppgiftsnamn"], + ["Input folder", "Eingangsordner", "Dossier source", "Invoermap", "Carpeta de entrada", "Inndatamappe", "Indatamapp"], + ["Output folder", "Ausgabeordner", "Dossier de sortie", "Uitvoermap", "Carpeta de salida", "Utdatamappe", "Utdatamapp"], + ["Archive folder", "Archivordner", "Dossier d’archivage", "Archiefmap", "Carpeta de archivo", "Arkivmappe", "Arkivmapp"], + ["Browse…", "Durchsuchen…", "Parcourir…", "Bladeren…", "Examinar…", "Bla gjennom…", "Bläddra…"], + [ + "Include subfolders", + "Unterordner einbeziehen", + "Inclure les sous-dossiers", + "Submappen opnemen", + "Incluir subcarpetas", + "Ta med undermapper", + "Inkludera undermappar", + ], + [ + "Images (PNG, JPEG, TIFF, BMP)", + "Bilder (PNG, JPEG, TIFF, BMP)", + "Images (PNG, JPEG, TIFF, BMP)", + "Afbeeldingen (PNG, JPEG, TIFF, BMP)", + "Imágenes (PNG, JPEG, TIFF, BMP)", + "Bilder (PNG, JPEG, TIFF, BMP)", + "Bilder (PNG, JPEG, TIFF, BMP)", + ], + ["Schedule", "Zeitplan", "Planification", "Planning", "Programación", "Tidsplan", "Schema"], + ["Run once", "Einmal ausführen", "Exécuter une fois", "Eenmalig uitvoeren", "Ejecutar una vez", "Kjør én gang", "Kör en gång"], + ["Recurring", "Wiederholt", "Récurrent", "Herhalen", "Periódica", "Gjentakende", "Återkommande"], + [ + "Start time (local)", + "Startzeit (lokal)", + "Heure de début (locale)", + "Starttijd (lokaal)", + "Hora de inicio (local)", + "Starttid (lokal)", + "Starttid (lokal)", + ], + [ + "Leave blank to start now; YYYY-MM-DD HH:MM", + "Leer lassen für sofort; JJJJ-MM-TT HH:MM", + "Vide pour démarrer maintenant ; AAAA-MM-JJ HH:MM", + "Leeg voor direct starten; JJJJ-MM-DD UU:MM", + "Vacío para iniciar ahora; AAAA-MM-DD HH:MM", + "Tomt for å starte nå; ÅÅÅÅ-MM-DD TT:MM", + "Lämna tomt för att börja nu; ÅÅÅÅ-MM-DD TT:MM", + ], + [ + "Check interval (seconds)", + "Prüfintervall (Sekunden)", + "Intervalle de vérification (secondes)", + "Controle-interval (seconden)", + "Intervalo de comprobación (segundos)", + "Kontrollintervall (sekunder)", + "Kontrollintervall (sekunder)", + ], + [ + "Wait for stable files (seconds)", + "Auf vollständige Dateien warten (Sekunden)", + "Attendre des fichiers stables (secondes)", + "Wachten op stabiele bestanden (seconden)", + "Esperar archivos estables (segundos)", + "Vent på stabile filer (sekunder)", + "Vänta på stabila filer (sekunder)", + ], + ["Original files", "Originaldateien", "Fichiers originaux", "Oorspronkelijke bestanden", "Archivos originales", "Originalfiler", "Originalfiler"], + ["Keep", "Behalten", "Conserver", "Bewaren", "Conservar", "Behold", "Behåll"], + [ + "Move after success", + "Nach Erfolg verschieben", + "Déplacer après réussite", + "Verplaatsen na succes", + "Mover tras completar", + "Flytt etter fullføring", + "Flytta efter slutförande", + ], + [ + "Delete after success", + "Nach Erfolg löschen", + "Supprimer après réussite", + "Verwijderen na succes", + "Eliminar tras completar", + "Slett etter fullføring", + "Ta bort efter slutförande", + ], + [ + "Only after the result is saved. Errors always keep the original.", + "Erst nach dem Speichern des Ergebnisses. Bei Fehlern bleibt das Original erhalten.", + "Uniquement après l’enregistrement du résultat. En cas d’erreur, l’original est conservé.", + "Pas nadat het resultaat is opgeslagen. Bij fouten blijft het origineel behouden.", + "Solo después de guardar el resultado. Los errores siempre conservan el original.", + "Kun etter at resultatet er lagret. Ved feil beholdes originalen.", + "Först när resultatet sparats. Vid fel behålls alltid originalet.", + ], + [ + "Scan and text recognition", + "Scan und Texterkennung", + "Numérisation et reconnaissance de texte", + "Scannen en tekstherkenning", + "Escaneo y reconocimiento de texto", + "Skanning og tekstgjenkjenning", + "Skanning och textigenkänning", + ], + ["Document language", "Dokumentsprache", "Langue du document", "Documenttaal", "Idioma del documento", "Dokumentspråk", "Dokumentspråk"], + ["Resolution (dpi)", "Auflösung (dpi)", "Résolution (ppp)", "Resolutie (dpi)", "Resolución (ppp)", "Oppløsning (dpi)", "Upplösning (dpi)"], + [ + "Add searchable text", + "Durchsuchbaren Text hinzufügen", + "Ajouter du texte consultable", + "Doorzoekbare tekst toevoegen", + "Añadir texto buscable", + "Legg til søkbar tekst", + "Lägg till sökbar text", + ], + [ + "Skip pages that already contain text", + "Seiten mit vorhandenem Text überspringen", + "Ignorer les pages contenant déjà du texte", + "Pagina’s met tekst overslaan", + "Omitir páginas que ya contienen texto", + "Hopp over sider som allerede har tekst", + "Hoppa över sidor som redan innehåller text", + ], + [ + "Automatically correct page orientation", + "Seitenausrichtung automatisch korrigieren", + "Corriger automatiquement l’orientation", + "Paginaoriëntatie automatisch corrigeren", + "Corregir orientación automáticamente", + "Korriger sideretningen automatisk", + "Korrigera sidriktningen automatiskt", + ], + [ + "Straighten skewed scans", + "Schief eingescannte Seiten geraderichten", + "Redresser les pages inclinées", + "Scheve scans rechtzetten", + "Enderezar escaneos inclinados", + "Rett opp skjeve skanninger", + "Räta upp sneda skanningar", + ], + [ + "Crop to detected paper edges", + "Auf erkannte Papierkanten beschneiden", + "Recadrer aux bords du papier détectés", + "Bijsnijden tot gedetecteerde papierranden", + "Recortar a los bordes del papel detectados", + "Beskjær til oppdagede papirkanter", + "Beskär till upptäckta papperskanter", + ], + [ + "Safety margin (pt)", + "Sicherheitsrand (pt)", + "Marge de sécurité (pt)", + "Veiligheidsmarge (pt)", + "Margen de seguridad (pt)", + "Sikkerhetsmargin (pt)", + "Säkerhetsmarginal (pt)", + ], + [ + "Uncertain orientations and paper edges are kept unchanged.", + "Unsichere Ausrichtungen und Papierkanten bleiben unverändert.", + "Les orientations et bords incertains restent inchangés.", + "Onzekere oriëntaties en papierranden blijven ongewijzigd.", + "Las orientaciones y bordes inciertos se conservan.", + "Usikre retninger og papirkanter beholdes uendret.", + "Osäkra riktningar och papperskanter lämnas oförändrade.", + ], + ["Watching", "Überwacht", "Surveillance", "Bewaken", "Supervisando", "Overvåker", "Bevakar"], + ["Stopped", "Gestoppt", "Arrêté", "Gestopt", "Detenida", "Stoppet", "Stoppad"], + ["Scheduled", "Geplant", "Planifié", "Gepland", "Programada", "Planlagt", "Schemalagd"], + ["Completed", "Abgeschlossen", "Terminé", "Voltooid", "Completada", "Fullført", "Slutförd"], + ["Waiting", "Wartend", "En attente", "Wachtend", "En espera", "Venter", "Väntar"], + ["Failed", "Fehlgeschlagen", "Échecs", "Mislukt", "Fallidos", "Mislykket", "Misslyckade"], + ["Error details", "Fehlerdetails", "Détails de l’erreur", "Foutdetails", "Detalles del error", "Feildetaljer", "Felinformation"], + ["Saved", "Gespeichert", "Enregistré", "Opgeslagen", "Guardado", "Lagret", "Sparat"], + [ + "Stop this task before editing.", + "Diese Aufgabe vor dem Bearbeiten stoppen.", + "Arrêtez cette tâche avant de la modifier.", + "Stop deze taak voordat u deze bewerkt.", + "Detenga esta tarea antes de editarla.", + "Stopp denne oppgaven før redigering.", + "Stoppa uppgiften innan du redigerar den.", + ], + [ + "Create a task to process new scans automatically.", + "Eine Aufgabe anlegen, um neue Scans automatisch zu verarbeiten.", + "Créez une tâche pour traiter automatiquement les nouveaux scans.", + "Maak een taak om nieuwe scans automatisch te verwerken.", + "Cree una tarea para procesar nuevos escaneos automáticamente.", + "Opprett en oppgave for å behandle nye skanninger automatisk.", + "Skapa en uppgift för att bearbeta nya skanningar automatiskt.", + ], + [ + "Processing continues when this window or PdfCraft is closed. Stop tasks here to pause it.", + "Die Verarbeitung läuft beim Schließen dieses Fensters oder von PdfCraft weiter. Hier stoppen, um sie anzuhalten.", + "Le traitement continue après fermeture de cette fenêtre ou de PdfCraft. Arrêtez les tâches ici pour le suspendre.", + "Verwerking gaat door wanneer dit venster of PdfCraft sluit. Stop taken hier om te pauzeren.", + "El proceso continúa al cerrar esta ventana o PdfCraft. Detenga las tareas aquí para pausarlo.", + "Behandlingen fortsetter når vinduet eller PdfCraft lukkes. Stopp oppgavene her for å pause.", + "Bearbetningen fortsätter när fönstret eller PdfCraft stängs. Stoppa uppgifterna här för att pausa.", + ], + [ + "OCR models are missing. Reinstall PdfCraft with OCR models.", + "OCR-Modelle fehlen. PdfCraft mit OCR-Modellen neu installieren.", + "Modèles OCR absents. Réinstallez PdfCraft avec les modèles OCR.", + "OCR-modellen ontbreken. Installeer PdfCraft opnieuw met OCR-modellen.", + "Faltan modelos OCR. Reinstale PdfCraft con modelos OCR.", + "OCR-modeller mangler. Installer PdfCraft med OCR-modeller på nytt.", + "OCR-modeller saknas. Installera om PdfCraft med OCR-modeller.", + ], + [ + "Invalid or ambiguous local start time", + "Ungültige oder mehrdeutige lokale Startzeit", + "Heure de début locale incorrecte ou ambiguë", + "Ongeldige of dubbelzinnige lokale starttijd", + "Hora local no válida o ambigua", + "Ugyldig eller tvetydig lokal starttid", + "Ogiltig eller tvetydig lokal starttid", + ], + ["Language", "Sprache", "Langue", "Taal", "Idioma", "Språk", "Språk"], +]; + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn hot_folder_seven_catalogs_are_complete_and_system_defaults_are_deterministic() { + let keys: std::collections::BTreeSet<_> = MESSAGES.iter().map(|row| row[0]).collect(); + assert_eq!(keys.len(), MESSAGES.len(), "duplicate translation keys"); + for row in MESSAGES { + assert!(row.iter().all(|text| !text.is_empty())); + } + let source = concat!(include_str!("ui.rs"), include_str!("source_ui.rs")); + for (at, _) in source.match_indices("t(\"") { + if source.get(..at).and_then(|s| s.chars().last()).is_some_and(|c| c.is_alphanumeric() || c == '_') { + continue; + } + let suffix = source.get(at + 3..).unwrap(); + let key = suffix.split('"').next().unwrap(); + assert!(keys.contains(key), "missing translation: {key}"); + } + for (locale, index) in [("de-AT", 1), ("fr-CA", 2), ("nl-NL", 3), ("es-ES", 4), ("no-NO", 5), ("sv-SE", 6), ("ja-JP", 0)] { + assert_eq!(Language::system(Some(locale)), Language(index)); + } + } +} diff --git a/apps/pdfcraft-hot-folder/src/main.rs b/apps/pdfcraft-hot-folder/src/main.rs new file mode 100644 index 0000000000..ee78cf31e3 --- /dev/null +++ b/apps/pdfcraft-hot-folder/src/main.rs @@ -0,0 +1,139 @@ +//! Optional standalone Hot Folder app. Nothing installs or enables it implicitly. +#![cfg_attr(target_os = "windows", windows_subsystem = "windows")] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod i18n; +mod source_ui; +mod startup; +mod ui; +mod worker; + +use pdfcraft_engine::hot_folder as hf; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +fn default_config() -> Result { + #[cfg(target_os = "windows")] + let path = std::env::var_os("LOCALAPPDATA").map(PathBuf::from).map(|p| p.join("PdfCraft/HotFolder/tasks.json")); + #[cfg(target_os = "macos")] + let path = std::env::var_os("HOME").map(PathBuf::from).map(|p| p.join("Library/Application Support/PdfCraft/HotFolder/tasks.json")); + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + let path = std::env::var_os("XDG_CONFIG_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(PathBuf::from).map(|p| p.join(".config"))) + .map(|p| p.join("pdfcraft/hot-folder/tasks.json")); + path.ok_or_else(|| "the user configuration directory is unavailable; use --config ".into()) +} + +fn launch_worker(config: &Path) -> Result<(), String> { + let executable = std::env::current_exe().map_err(|e| e.to_string())?; + if pdfcraft_hot_folder_service::status(&executable, config)?.installed { + return pdfcraft_hot_folder_service::start(&executable, config); + } + worker::check_identity()?; + let mut command = Command::new(executable); + command.arg("--worker").arg("--config").arg(config).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x08000000); // CREATE_NO_WINDOW; no visible command window. + } + command.spawn().map_err(|e| e.to_string())?; + Ok(()) +} + +fn run() -> Result<(), String> { + let mut args = std::env::args().skip(1); + let mut path = None; + let mut background = false; + let mut once = false; + let mut service = false; + let mut service_action = None; + let mut account = String::new(); + let mut initialize = false; + while let Some(arg) = args.next() { + match arg.as_str() { + "--config" => path = Some(PathBuf::from(args.next().ok_or("--config needs a file path")?)), + "--worker" => background = true, + "--once" => once = true, + "--service" => service = true, + "--initialize-config" => initialize = true, + "--service-action" => { + service_action = Some(args.next().ok_or("--service-action needs status/start/stop/enable/disable/install/uninstall")?) + } + "--account" => account = args.next().ok_or("--account needs a non-root service account")?, + "--version" => { + println!("PdfCraft Hot Folder {}", env!("CARGO_PKG_VERSION")); + return Ok(()); + } + "--help" | "-h" => { + println!( + "PdfCraft Hot Folder\n pdfcraft-hot-folder [--config FILE]\n pdfcraft-hot-folder --worker --config FILE\n pdfcraft-hot-folder --once --config FILE\n pdfcraft-hot-folder --service --config FILE\n pdfcraft-hot-folder --initialize-config --config FILE\n pdfcraft-hot-folder --service-action status|start|stop|enable|disable|install|uninstall --config FILE [--account USER]\nThe GUI saves opt-in tasks. --once runs one scheduler pass; --service hosts a registered boot service. No network listener is opened." + ); + return Ok(()); + } + _ => return Err(format!("unknown option: {arg}")), + } + } + let config = path.map(Ok).unwrap_or_else(default_config)?; + let config = if config.is_absolute() { config } else { std::env::current_dir().map_err(|e| e.to_string())?.join(config) }; + if usize::from(service) + usize::from(once) + usize::from(background) + usize::from(service_action.is_some()) + usize::from(initialize) > 1 { + return Err("choose only one operating mode".into()); + } + if initialize { + return hf::initialize_config(&config); + } + if let Some(action) = service_action { + let exe = std::env::current_exe().map_err(|e| e.to_string())?; + use pdfcraft_hot_folder_service as svc; + return match action.as_str() { + "status" => { + println!("{}", serde_json::to_string_pretty(&svc::status(&exe, &config)?).map_err(|e| e.to_string())?); + Ok(()) + } + "start" => svc::start(&exe, &config), + "stop" => svc::stop(&exe, &config), + "enable" => svc::set_enabled(true, &exe, &config), + "disable" => svc::set_enabled(false, &exe, &config), + "install" => svc::install(&exe, &config, &account), + "uninstall" => svc::uninstall(&exe, &config), + _ => Err("unknown service action; use status/start/stop/enable/disable/install/uninstall".into()), + }; + } + if service { + return pdfcraft_hot_folder_service::run(&config, |path, stop| worker::run(path, || stop.load(std::sync::atomic::Ordering::Relaxed))); + } + if once { + worker::check_identity()?; + let state = hf::poll(&config, None)?; + println!("{}", serde_json::to_string_pretty(&state).map_err(|e| e.to_string())?); + return Ok(()); + } + if background { + return worker::run(&config, || false); + } + let options = eframe::NativeOptions { + viewport: egui::ViewportBuilder::default().with_inner_size([1080.0, 780.0]).with_min_inner_size([780.0, 560.0]), + renderer: eframe::Renderer::Glow, + ..Default::default() + }; + eframe::run_native( + "PdfCraft Hot Folder", + options, + Box::new(move |cc| { + ui::install_fonts(&cc.egui_ctx); + Ok(Box::new(ui::HotFolderApp::new(config))) + }), + ) + .map_err(|e| e.to_string()) +} + +fn main() -> std::process::ExitCode { + match run() { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(e) => { + eprintln!("PdfCraft Hot Folder: {e}"); + std::process::ExitCode::FAILURE + } + } +} diff --git a/apps/pdfcraft-hot-folder/src/source_ui.rs b/apps/pdfcraft-hot-folder/src/source_ui.rs new file mode 100644 index 0000000000..3667885699 --- /dev/null +++ b/apps/pdfcraft-hot-folder/src/source_ui.rs @@ -0,0 +1,307 @@ +//! Source settings and explicit asynchronous authentication. Passwords never enter task JSON. +use crate::i18n::Language; +use pdfcraft_engine::{hot_folder as hf, hot_folder_sources as source}; +use std::path::Path; +use std::sync::mpsc::{self, Receiver}; +use std::time::{Duration, Instant}; +use zeroize::Zeroizing; + +enum Job { + Password(String, Zeroizing), + Forget(String), + Status(String), + Begin(source::OutlookSource), + Poll(source::OutlookSource, source::DeviceSignIn), + Folders(source::OutlookSource, Option), +} +enum Event { + Saved, + Forgotten, + Status(bool), + Begun(source::OutlookSource, source::DeviceSignIn), + Polled(source::SignInStatus), + Folders(Vec), +} +struct SignIn { + source: source::OutlookSource, + code: source::DeviceSignIn, + next: Instant, +} +#[derive(Default)] +pub struct SourceUi { + password: String, + pending: Option, + receiver: Option>>, + sign_in: Option, + pub error: Option, + message: Option<&'static str>, + folders: Vec, +} +impl Drop for SourceUi { + fn drop(&mut self) { + use zeroize::Zeroize; + self.password.zeroize(); + } +} +impl SourceUi { + pub fn has_pending(&self) -> bool { + self.pending.is_some() + } + pub fn cancel_pending(&mut self) { + self.pending = None; + } + pub fn begin_pending(&mut self, config: &Path) -> Result<(), String> { + let Some(job) = self.pending.take() else { return Ok(()) }; + let store = source::CredentialStore::for_config(config, None)?; + self.spawn(job, store); + Ok(()) + } + fn spawn(&mut self, job: Job, store: source::CredentialStore) { + let (tx, rx) = mpsc::channel(); + self.receiver = Some(rx); + self.error = None; + self.message = None; + std::thread::spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| match job { + Job::Password(id, password) => source::save_ftp_password(&store, &id, &password).map(|_| Event::Saved), + Job::Forget(id) => source::forget_credentials(&store, &id).map(|_| Event::Forgotten), + Job::Status(id) => source::credential_status(&store, &id).map(Event::Status), + Job::Begin(settings) => source::outlook_begin_sign_in(&store, &settings).map(|code| Event::Begun(settings, code)), + Job::Poll(settings, code) => source::outlook_poll_sign_in(&store, &settings, &code).map(Event::Polled), + Job::Folders(settings, parent) => source::outlook_folders(&store, &settings, parent.as_deref(), &|| false).map(Event::Folders), + })) + .unwrap_or_else(|_| Err("source authentication failed safely".into())); + let _ = tx.send(result); + }); + } + pub fn tick(&mut self, config: &Path) { + let result = self.receiver.as_ref().and_then(|rx| match rx.try_recv() { + Ok(value) => Some(value), + Err(mpsc::TryRecvError::Empty) => None, + Err(_) => Some(Err("source worker stopped unexpectedly".into())), + }); + if let Some(result) = result { + self.receiver = None; + match result { + Err(error) => { + self.error = Some(error); + self.sign_in = None; + } + Ok(Event::Saved) => self.message = Some("Credentials saved"), + Ok(Event::Forgotten) => { + self.message = Some("Credentials removed"); + self.sign_in = None; + } + Ok(Event::Status(found)) => self.message = Some(if found { "Credentials available" } else { "Sign-in required" }), + Ok(Event::Begun(settings, code)) => { + self.sign_in = Some(SignIn { next: Instant::now() + Duration::from_secs(code.interval_seconds.max(1)), source: settings, code }); + } + Ok(Event::Polled(source::SignInStatus::Pending { retry_after_seconds })) => { + if let Some(sign) = &mut self.sign_in { + sign.next = Instant::now() + Duration::from_secs(retry_after_seconds.max(1)); + } + } + Ok(Event::Polled(status)) => { + self.sign_in = None; + self.message = Some(match status { + source::SignInStatus::Authorized => "Signed in", + source::SignInStatus::Expired => "Sign-in expired", + _ => "Sign-in declined", + }); + } + Ok(Event::Folders(folders)) => self.folders = folders, + } + } + if self.receiver.is_none() + && let Some(sign) = &self.sign_in + && Instant::now() >= sign.next + { + let job = Job::Poll(sign.source.clone(), sign.code.clone()); + match source::CredentialStore::for_config(config, None) { + Ok(store) => self.spawn(job, store), + Err(error) => { + self.error = Some(error); + self.sign_in = None; + } + } + } + } + pub fn show(&mut self, ui: &mut egui::Ui, task: &mut hf::Task, lang: Language) { + let t = |key| lang.t(key); + ui.heading(t("Source")); + let mut kind = match &task.source { + source::SourceConfig::Local => 0, + source::SourceConfig::Ftp(_) => 1, + source::SourceConfig::Outlook(_) => 2, + source::SourceConfig::Project(_) => 3, + }; + let old = kind; + let names = ["Local or network folder", "FTP / FTPS", "Outlook attachments", "PDFCraft project"]; + egui::ComboBox::from_id_salt("source-kind").selected_text(t(names.get(kind).copied().unwrap_or(names[0]))).show_ui(ui, |ui| { + for (index, name) in names.iter().enumerate() { + ui.selectable_value(&mut kind, index, t(name)); + } + }); + if kind != old { + task.source = match kind { + 1 => source::SourceConfig::Ftp(source::FtpSource { credential_id: format!("{}-ftp", task.id), ..Default::default() }), + 2 => source::SourceConfig::Outlook(source::OutlookSource { credential_id: format!("{}-mail", task.id), ..Default::default() }), + 3 => source::SourceConfig::Project(Default::default()), + _ => source::SourceConfig::Local, + }; + task.originals = hf::Originals::Keep; + self.folders.clear(); + } + ui.add_enabled_ui(self.receiver.is_none(), |ui| match &mut task.source { + source::SourceConfig::Local => {} + source::SourceConfig::Ftp(settings) => { + line(ui, t("Server"), &mut settings.host); + ui.horizontal(|ui| { + ui.label(t("Port")); + ui.add(egui::DragValue::new(&mut settings.port).range(1..=65535)); + }); + ui.horizontal_wrapped(|ui| { + ui.radio_value(&mut settings.security, source::FtpSecurity::ExplicitTls, t("FTP with TLS")); + ui.radio_value(&mut settings.security, source::FtpSecurity::Plain, t("Unencrypted FTP")); + }); + line(ui, t("Username"), &mut settings.username); + line(ui, t("Server folder"), &mut settings.folder); + ui.checkbox(&mut settings.recursive, t("Include subfolders")); + line(ui, t("Credential ID"), &mut settings.credential_id); + ui.label(t("Password")); + ui.add(egui::TextEdit::singleline(&mut self.password).password(true)); + if ui.button(t("Save password securely")).clicked() { + self.pending = Some(Job::Password(settings.credential_id.clone(), Zeroizing::new(std::mem::take(&mut self.password)))); + } + self.credential_buttons(ui, &settings.credential_id, lang); + action(ui, &mut settings.action, t("Archive server folder"), lang); + } + source::SourceConfig::Outlook(settings) => { + line(ui, t("Application ID (public client)"), &mut settings.client_id); + line(ui, t("Tenant"), &mut settings.tenant); + line(ui, t("Credential ID"), &mut settings.credential_id); + ui.weak(t("Use your own Microsoft application registration with public-client device sign-in enabled.")); + ui.horizontal_wrapped(|ui| { + if ui.button(t("Sign in to Outlook")).clicked() { + self.pending = Some(Job::Begin(settings.clone())); + } + if ui.button(t("Load mail folders")).clicked() { + self.pending = Some(Job::Folders(settings.clone(), None)); + } + }); + self.credential_buttons(ui, &settings.credential_id, lang); + line(ui, t("Mail folder ID"), &mut settings.folder_id); + for folder in &self.folders { + ui.horizontal(|ui| { + if ui.selectable_label(settings.folder_id == folder.id, &folder.name).clicked() { + settings.folder_id = folder.id.clone(); + } + if folder.child_count > 0 && ui.button(t("Open subfolders")).clicked() { + self.pending = Some(Job::Folders(settings.clone(), Some(folder.id.clone()))); + } + }); + } + action(ui, &mut settings.action, t("Archive mail folder ID"), lang); + ui.weak(t("Mail move/delete affects the whole message and waits for all supported attachments. Enable PDF and images.")); + } + source::SourceConfig::Project(settings) => { + let mut path = settings.manifest.to_string_lossy().into_owned(); + line(ui, t("Project manifest"), &mut path); + settings.manifest = path.into(); + ui.horizontal_wrapped(|ui| { + if ui.button(t("Choose project…")).clicked() + && let Some(path) = rfd::FileDialog::new().add_filter("JSON", &["json"]).pick_file() + { + settings.manifest = path; + } + if ui.button(t("Create project from files…")).clicked() { + match create_project() { + Ok(Some(path)) => settings.manifest = path, + Ok(None) => {} + Err(error) => self.error = Some(error), + } + } + }); + ui.weak(t("Open PDFCraft manifests contain an ordered list of PDF/image files. Proprietary ABBYY projects are not supported.")); + action(ui, &mut settings.action, t("Archive folder"), lang); + } + }); + if self.receiver.is_some() { + ui.spinner(); + ui.label(t("Connecting…")); + } + if let Some(sign) = &self.sign_in { + ui.label(t("Open the sign-in page and enter this code:")); + ui.hyperlink_to(&sign.code.verification_uri, &sign.code.verification_uri); + ui.monospace(&sign.code.user_code); + } + if let Some(message) = self.message { + ui.label(t(message)); + } + if let Some(error) = &self.error { + ui.colored_label(egui::Color32::DARK_RED, t("Error details")); + ui.label(error); + } + } + fn credential_buttons(&mut self, ui: &mut egui::Ui, id: &str, lang: Language) { + ui.horizontal_wrapped(|ui| { + if ui.button(lang.t("Check credentials")).clicked() { + self.pending = Some(Job::Status(id.into())); + } + if ui.button(lang.t("Remove credentials")).clicked() { + self.pending = Some(Job::Forget(id.into())); + } + }); + ui.weak(lang.t("Credentials stay in the protected service store and are never exported with tasks.")); + } +} +fn line(ui: &mut egui::Ui, label: &str, value: &mut String) { + ui.label(label); + ui.text_edit_singleline(value); +} +fn action(ui: &mut egui::Ui, value: &mut source::SourceAction, destination_label: &str, lang: Language) { + let t = |key| lang.t(key); + ui.label(t("After processing originals")); + let mut kind = match value { + source::SourceAction::Keep => 0, + source::SourceAction::Move { .. } => 1, + source::SourceAction::Delete => 2, + }; + let old = kind; + ui.horizontal_wrapped(|ui| { + for (index, name) in ["Keep", "Move after success", "Delete after success"].iter().enumerate() { + ui.radio_value(&mut kind, index, t(name)); + } + }); + if old != kind { + *value = match kind { + 1 => source::SourceAction::Move { destination: String::new() }, + 2 => source::SourceAction::Delete, + _ => source::SourceAction::Keep, + }; + } + if let source::SourceAction::Move { destination } = value { + line(ui, destination_label, destination); + } +} +fn create_project() -> Result, String> { + let Some(files) = rfd::FileDialog::new().add_filter("PDF / images", &["pdf", "png", "jpg", "jpeg", "tif", "tiff"]).pick_files() else { + return Ok(None); + }; + let Some(first) = files.first().and_then(|p| p.parent()) else { return Ok(None) }; + let Some(path) = rfd::FileDialog::new().set_directory(first).set_file_name("hot-folder-project.json").add_filter("JSON", &["json"]).save_file() + else { + return Ok(None); + }; + let parent = path.parent().ok_or("project needs a parent folder")?; + let relative = files + .iter() + .map(|file| { + file.strip_prefix(parent) + .map(Path::to_path_buf) + .map_err(|_| "save the project in a folder containing all selected source files".to_owned()) + }) + .collect::, _>>()?; + source::write_project_manifest(&path, &source::ProjectManifest { files: relative, ..Default::default() }, None)?; + Ok(Some(path)) +} diff --git a/apps/pdfcraft-hot-folder/src/startup.rs b/apps/pdfcraft-hot-folder/src/startup.rs new file mode 100644 index 0000000000..37312407ae --- /dev/null +++ b/apps/pdfcraft-hot-folder/src/startup.rs @@ -0,0 +1,10 @@ +//! System-boot startup backed by Windows SCM, systemd, or a macOS LaunchDaemon. +use std::path::Path; + +pub fn enabled(executable: &Path, config: &Path) -> bool { + pdfcraft_hot_folder_service::status(executable, config).is_ok_and(|state| state.enabled) +} + +pub fn set_enabled(enabled: bool, executable: &Path, config: &Path) -> Result<(), String> { + pdfcraft_hot_folder_service::set_enabled(enabled, executable, config) +} diff --git a/apps/pdfcraft-hot-folder/src/ui.rs b/apps/pdfcraft-hot-folder/src/ui.rs new file mode 100644 index 0000000000..de1e9f2768 --- /dev/null +++ b/apps/pdfcraft-hot-folder/src/ui.rs @@ -0,0 +1,1030 @@ +use crate::i18n::{LANGUAGES, Language}; +use chrono::{Local, TimeZone}; +use pdfcraft_engine::{hot_folder as hf, hot_folder_output as output, ocr}; +use pdfcraft_hot_folder_service as service; +use std::path::PathBuf; + +pub fn install_fonts(ctx: &egui::Context) { + let mut fonts = egui::FontDefinitions::default(); + fonts + .font_data + .insert("Inter".into(), std::sync::Arc::new(egui::FontData::from_static(include_bytes!("../../../assets/fonts/Inter-Regular.ttf")))); + fonts.families.entry(egui::FontFamily::Proportional).or_default().insert(0, "Inter".into()); + for face in pdfcraft_fonts::ui_cjk_fonts(false) { + let name = face.name(); + fonts.font_data.insert(name.clone(), std::sync::Arc::new(egui::FontData::from_static(face.bytes))); + fonts.families.entry(egui::FontFamily::Proportional).or_default().push(name); + } + ctx.set_fonts(fonts); + ctx.set_visuals(egui::Visuals::light()); +} + +pub struct HotFolderApp { + path: PathBuf, + config: hf::Config, + state: hf::State, + selected: usize, + start_text: String, + language: Language, + error: Option, + saved: bool, + last_refresh: std::time::Instant, + errors_only: bool, + startup_enabled: bool, + service_state: Option, + service_account: String, + load_error: Option, + saved_snapshot: Vec, + disk_snapshot: Option>, + close_prompt: bool, + allow_close: bool, + source_ui: crate::source_ui::SourceUi, + /// Tests never launch an external process. + #[cfg(test)] + inline: bool, +} + +impl HotFolderApp { + pub fn new(path: PathBuf) -> Self { + let loaded = if path.exists() { hf::load_config(&path) } else { Ok(hf::Config::default()) }; + let error = loaded.as_ref().err().cloned(); + let saved_snapshot = loaded.as_ref().ok().and_then(|c| serde_json::to_vec(c).ok()).unwrap_or_default(); + let disk_snapshot = read_disk(&path).ok().flatten(); + let startup_enabled = std::env::current_exe().is_ok_and(|exe| crate::startup::enabled(&exe, &path)); + let service_state = std::env::current_exe().map_err(|e| e.to_string()).and_then(|exe| service::status(&exe, &path)); + let initial_error = error.clone().or_else(|| service_state.as_ref().err().cloned()); + let mut app = Self { + state: hf::read_state(&path).unwrap_or_default(), + path: path.clone(), + config: loaded.unwrap_or_default(), + selected: usize::MAX, + start_text: String::new(), + language: Language::system(sys_locale::get_locale().as_deref()), + load_error: error.clone(), + error: initial_error, + saved: false, + saved_snapshot, + disk_snapshot, + close_prompt: false, + allow_close: false, + source_ui: Default::default(), + last_refresh: std::time::Instant::now(), + errors_only: false, + startup_enabled, + service_state: service_state.ok(), + service_account: String::new(), + #[cfg(test)] + inline: false, + }; + app.select(0); + app + } + + fn select(&mut self, index: usize) { + if let Err(error) = self.update_start() { + self.error = Some(error); + return; + } + self.selected = index; + self.source_ui = Default::default(); + self.start_text = self + .config + .tasks + .get(index) + .and_then(|t| (t.start_at > 0).then_some(t.start_at)) + .and_then(|v| i64::try_from(v).ok()) + .and_then(|v| Local.timestamp_opt(v, 0).single()) + .map(|d| d.format("%Y-%m-%d %H:%M").to_string()) + .unwrap_or_default(); + } + + fn new_task(&mut self, copy: bool) { + let mut task = if copy { self.config.tasks.get(self.selected).cloned().unwrap_or_default() } else { hf::Task::default() }; + let micros = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_micros()); + task.id = format!("task-{micros}-{}", self.config.tasks.len()); + task.enabled = false; + if !copy { + task.name = self.language.t("New task").into(); + task.settings.language = ocr::language_for_locale(sys_locale::get_locale().as_deref()).into(); + } + self.config.tasks.push(task); + self.select(self.config.tasks.len().saturating_sub(1)); + self.saved = false; + } + + fn update_start(&mut self) -> Result<(), String> { + if let Some(task) = self.config.tasks.get_mut(self.selected) { + task.start_at = if self.start_text.trim().is_empty() { + 0 + } else { + let naive = chrono::NaiveDateTime::parse_from_str(self.start_text.trim(), "%Y-%m-%d %H:%M") + .map_err(|_| self.language.t("Invalid or ambiguous local start time"))?; + let value = Local.from_local_datetime(&naive).single().ok_or(self.language.t("Invalid or ambiguous local start time"))?.timestamp(); + u64::try_from(value).map_err(|_| self.language.t("Invalid or ambiguous local start time"))? + }; + } + Ok(()) + } + + fn dirty(&self) -> bool { + let saved = serde_json::to_vec(&self.config).ok(); + let date = self + .config + .tasks + .get(self.selected) + .and_then(|t| i64::try_from(t.start_at).ok()) + .filter(|t| *t > 0) + .and_then(|t| Local.timestamp_opt(t, 0).single()) + .map(|t| t.format("%Y-%m-%d %H:%M").to_string()) + .unwrap_or_default(); + saved.as_ref() != Some(&self.saved_snapshot) || date != self.start_text + } + + fn save(&mut self) -> Result<(), String> { + if let Some(error) = &self.load_error { + return Err(error.clone()); + } + self.update_start()?; + if read_disk(&self.path)? != self.disk_snapshot { + return Err(self.language.t("Settings changed in another window. Reload before saving.").into()); + } + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + hf::save_config(&self.path, &self.config, None)?; + self.saved = true; + self.saved_snapshot = serde_json::to_vec(&self.config).map_err(|e| e.to_string())?; + self.disk_snapshot = read_disk(&self.path)?; + Ok(()) + } + + fn start(&mut self, immediate: bool) -> Result<(), String> { + if immediate { + self.start_text.clear(); + } + self.save()?; + let id = self.config.tasks.get(self.selected).ok_or("no task selected")?.id.clone(); + if immediate { + hf::run_now(&self.path, &id, None)?; + } else { + hf::reset(&self.path, &id, None)?; + } + if let Some(task) = self.config.tasks.get_mut(self.selected) { + task.enabled = true; + } + self.save()?; + #[cfg(test)] + if self.inline { + return Ok(()); + } + if let Err(error) = crate::launch_worker(&self.path) { + if let Some(task) = self.config.tasks.get_mut(self.selected) { + task.enabled = false; + } + return match self.save() { + Ok(()) => Err(error), + Err(save) => Err(format!("{error}; {save}")), + }; + } + Ok(()) + } + + fn stop(&mut self, all: bool) -> Result<(), String> { + // Stop uses persisted settings, so an unrelated incomplete draft cannot prevent it. + let mut config = hf::load_config(&self.path)?; + let id = self.config.tasks.get(self.selected).map(|t| t.id.clone()); + for task in &mut config.tasks { + if all || Some(&task.id) == id.as_ref() { + task.enabled = false; + } + } + hf::save_config(&self.path, &config, None)?; + self.disk_snapshot = read_disk(&self.path)?; + self.saved_snapshot = serde_json::to_vec(&config).map_err(|e| e.to_string())?; + for task in &mut self.config.tasks { + if all || Some(&task.id) == id.as_ref() { + task.enabled = false; + } + } + Ok(()) + } + + fn refresh(&mut self) { + self.source_ui.tick(&self.path); + if self.last_refresh.elapsed().as_millis() < 1000 { + return; + } + self.last_refresh = std::time::Instant::now(); + match hf::read_state(&self.path) { + Ok(state) => self.state = state, + Err(e) => self.error = Some(e), + } + match hf::read_worker_error(&self.path) { + Ok(Some(error)) | Err(error) => self.error = Some(error), + Ok(None) => {} + } + } + + fn import_tasks(&mut self) -> Result<(), String> { + let Some(path) = rfd::FileDialog::new().add_filter("JSON", &["json"]).pick_file() else { + return Ok(()); + }; + let mut imported = hf::load_config(&path)?; + // Importing a task never starts it or deletes a source implicitly. + let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |v| v.as_micros()); + for (index, task) in imported.tasks.iter_mut().enumerate() { + task.enabled = false; + if self.config.tasks.iter().any(|existing| existing.id == task.id) { + task.id = format!("imported-{stamp}-{index}"); + } + } + hf::validate(&imported, None)?; + let mut merged = self.config.clone(); + merged.tasks.extend(imported.tasks); + hf::validate(&merged, None)?; + self.config = merged; + self.save() + } + + fn service_action(&mut self, action: &str) -> Result<(), String> { + let exe = std::env::current_exe().map_err(|e| e.to_string())?; + match action { + "start" => service::start(&exe, &self.path)?, + "stop" => service::stop(&exe, &self.path)?, + "enable" => crate::startup::set_enabled(true, &exe, &self.path)?, + "disable" => crate::startup::set_enabled(false, &exe, &self.path)?, + "install" => { + self.save()?; + service::install(&exe, &self.path, &self.service_account)?; + } + "uninstall" => service::uninstall(&exe, &self.path)?, + _ => {} + } + let status = service::status(&exe, &self.path)?; + self.startup_enabled = status.enabled; + self.service_state = Some(status); + Ok(()) + } + + pub fn show(&mut self, root: &mut egui::Ui) { + self.refresh(); + let ctx = root.ctx(); + ctx.request_repaint_after(std::time::Duration::from_secs(1)); + let lang = self.language; + let t = |key| lang.t(key); + egui::Panel::top("header").show(root, |ui| { + ui.add_space(8.0); + ui.horizontal(|ui| { + ui.heading("PdfCraft Hot Folder"); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + egui::ComboBox::from_id_salt("ui-language").selected_text(LANGUAGES.get(self.language.0).map_or("English", |l| l.1)).show_ui( + ui, + |ui| { + for (index, (_, name)) in LANGUAGES.iter().enumerate() { + ui.selectable_value(&mut self.language, Language(index), *name); + } + }, + ); + ui.label(t("Language")); + }); + }); + ui.label(t("Processing continues when this window or PdfCraft is closed. Stop tasks here to pause it.")); + ui.add_space(8.0); + }); + if let Some(error) = &self.error { + egui::Panel::top("error").show(root, |ui| { + ui.colored_label(egui::Color32::DARK_RED, t("Error details")); + egui::ScrollArea::vertical().id_salt("error-details").max_height(80.0).show(ui, |ui| { + ui.label(error); + }); + }); + } + egui::Panel::left("tasks").resizable(true).default_size(250.0).show(root, |ui| { + egui::ScrollArea::vertical().id_salt("task-list").show(ui, |ui| { + ui.heading(t("Tasks")); + if ui.add_enabled(self.config.tasks.len() < 32, egui::Button::new(t("New task"))).clicked() { + self.new_task(false); + } + if ui.button(t("Import tasks…")).clicked() { + self.error = self.import_tasks().err(); + } + if ui.button(t("Export tasks…")).clicked() + && let Some(path) = rfd::FileDialog::new().add_filter("JSON", &["json"]).set_file_name("hot-folder-tasks.json").save_file() + { + self.error = hf::save_config(&path, &self.config, None).err(); + } + if ui.button(t("Discard edits and reload")).clicked() { + let language = self.language; + *self = Self::new(self.path.clone()); + self.language = language; + } + ui.checkbox(&mut self.errors_only, t("Show errors only")); + let mut selected = self.selected; + for (index, task) in self.config.tasks.iter().enumerate() { + let status = self.state.tasks.get(&task.id); + if self.errors_only && status.is_none_or(|s| s.error.is_none()) { + continue; + } + let label = if !task.enabled { + "Stopped" + } else if status.is_some_and(|s| s.error.is_some()) { + "Failed" + } else if status.is_some_and(|s| s.finished) { + "Completed" + } else if task.start_at > hf::now() { + "Scheduled" + } else { + "Watching" + }; + ui.selectable_value(&mut selected, index, format!("{}\n{}", task.name, t(label))); + ui.small(task.input.display().to_string()); + if task.enabled { + let next = status.map_or(task.start_at, |s| s.next_check.max(task.start_at)); + if next > 0 + && let Ok(next) = i64::try_from(next) + && let Some(date) = Local.timestamp_opt(next, 0).single() + { + ui.small(format!("{}: {}", t("Next check"), date.format("%Y-%m-%d %H:%M:%S"))); + } + } + } + if selected != self.selected { + self.select(selected); + } + ui.separator(); + if ui.button(t("Stop all")).clicked() { + self.error = self.stop(true).err(); + } + ui.collapsing(t("System service"), |ui| { + let installed = self.service_state.as_ref().is_some_and(|s| s.installed); + let running = self.service_state.as_ref().is_some_and(|s| s.running); + ui.label(t(if self.service_state.is_none() { + "Status unavailable" + } else if running { + "Running" + } else if installed { + "Stopped" + } else { + "Not installed" + })); + ui.small(self.path.display().to_string()); + if let Some(state) = &self.service_state + && installed + { + ui.label(format!("{}: {}", t("Service account"), state.account)); + } + let mut action = None; + let mut enabled = self.startup_enabled; + if ui.add_enabled(installed, egui::Checkbox::new(&mut enabled, t("Start with the system"))).changed() { + action = Some(if enabled { "enable" } else { "disable" }); + } + if ui.add_enabled(installed && !running, egui::Button::new(t("Start service"))).clicked() { + action = Some("start"); + } + if ui.add_enabled(installed && running, egui::Button::new(t("Stop service"))).clicked() { + action = Some("stop"); + } + if ui.button(t("Refresh status")).clicked() { + action = Some("status"); + } + #[cfg(unix)] + if !installed { + ui.label(t("Service account")); + ui.text_edit_singleline(&mut self.service_account); + if ui.button(t("Install service")).clicked() { + action = Some("install"); + } + } + #[cfg(unix)] + if installed && ui.button(t("Uninstall service")).clicked() { + action = Some("uninstall"); + } + #[cfg(windows)] + ui.weak(t("Install or remove the system service with the optional Windows installer component.")); + ui.weak(t("Service changes require administrator rights. Grant the service account access to the selected folders.")); + ui.weak(t("To edit protected service settings, reopen Hot Folder as administrator with the same configuration path.")); + if let Some(action) = action { + self.error = self.service_action(action).err(); + } + }); + }); + }); + egui::CentralPanel::default().show(root, |ui| { + egui::ScrollArea::vertical().show(ui, |ui| { + let mut save = false; + let mut start = false; + let mut scheduled = false; + let mut stop = false; + let mut duplicate = false; + let mut delete = false; + let mut retry = false; + let can_add = self.config.tasks.len() < 32; + if let Some(task) = self.config.tasks.get_mut(self.selected) { + ui.horizontal_wrapped(|ui| { + save = ui.add_enabled(!task.enabled, egui::Button::new(t("Save"))).clicked(); + start = ui.add_enabled(!task.enabled && (ocr::available() || !needs_ocr(task)), egui::Button::new(t("Start now"))).clicked(); + scheduled = ui + .add_enabled(!task.enabled && (ocr::available() || !needs_ocr(task)), egui::Button::new(t("Start on schedule"))) + .clicked(); + stop = ui.add_enabled(task.enabled, egui::Button::new(t("Stop"))).clicked(); + duplicate = ui.add_enabled(can_add, egui::Button::new(t("Duplicate"))).clicked(); + delete = ui.add_enabled(!task.enabled, egui::Button::new(t("Delete task"))).clicked(); + }); + if task.enabled { + ui.label(t("Stop this task before editing.")); + } + if !ocr::available() && needs_ocr(task) { + ui.colored_label(egui::Color32::DARK_RED, t("OCR models are missing. Reinstall PdfCraft with OCR models.")); + } + ui.add_enabled_ui(!task.enabled, |ui| { + ui.label(t("Task name")); + ui.text_edit_singleline(&mut task.name); + self.source_ui.show(ui, task, lang); + let local = matches!(task.source, pdfcraft_engine::hot_folder_sources::SourceConfig::Local); + folder(ui, &mut task.input, t(if local { "Input folder" } else { "Staging folder" }), lang); + folder(ui, &mut task.output, t("Output folder"), lang); + if local { + ui.checkbox(&mut task.recursive, t("Include subfolders")); + } + ui.horizontal(|ui| { + ui.checkbox(&mut task.pdf, "PDF"); + ui.checkbox(&mut task.images, t(if local { "Images (PNG, JPEG, TIFF, BMP)" } else { "Images (PNG, JPEG, TIFF)" })); + }); + ui.checkbox(&mut task.analysis_only, t("Analyze without OCR")); + if task.analysis_only { + ui.weak(t("Creates JSON layout reports. OCR, image changes and blank-page removal are disabled.")); + } + output_steps(ui, &mut task.output_plan, lang, task.analysis_only); + ui.separator(); + ui.heading(t("Schedule")); + ui.horizontal(|ui| { + ui.radio_value(&mut task.once, true, t("Run once")); + ui.radio_value(&mut task.once, false, t("Recurring")); + }); + if !task.once { + calendar(ui, &mut task.schedule, lang); + } + ui.label(t("Start time (local)")); + ui.text_edit_singleline(&mut self.start_text); + ui.weak(t("Leave blank to start now; YYYY-MM-DD HH:MM")); + ui.horizontal(|ui| { + ui.label(t("Check interval (seconds)")); + ui.add(egui::DragValue::new(&mut task.interval_seconds).range(1..=86400)); + }); + ui.horizontal(|ui| { + ui.label(t("Wait for stable files (seconds)")); + ui.add(egui::DragValue::new(&mut task.settle_seconds).range(1..=300)); + }); + ui.separator(); + if local { + ui.heading(t("Original files")); + ui.horizontal_wrapped(|ui| { + ui.radio_value(&mut task.originals, hf::Originals::Keep, t("Keep")); + ui.radio_value(&mut task.originals, hf::Originals::Move, t("Move after success")); + ui.radio_value(&mut task.originals, hf::Originals::Delete, t("Delete after success")); + }); + if task.originals == hf::Originals::Move { + folder(ui, &mut task.archive, t("Archive folder"), lang); + } + ui.weak(t("Only after the result is saved. Errors always keep the original.")); + } + ui.separator(); + if task.analysis_only { + ui.label(t("Pages to process (empty = all; example: 1-3,5)")); + ui.text_edit_singleline(&mut task.preprocessing.pages); + } else { + ui.heading(t("Scan and text recognition")); + ui.horizontal(|ui| { + ui.label(t("Document language")); + let name = ocr::LANGUAGES.iter().find(|l| l.0 == task.settings.language).map_or("English", |l| l.1); + egui::ComboBox::from_id_salt("ocr-language").height(480.0).selected_text(name).show_ui(ui, |ui| { + for (code, name) in ocr::LANGUAGES { + ui.selectable_value(&mut task.settings.language, (*code).into(), *name); + } + }); + }); + ui.horizontal(|ui| { + ui.label(t("Resolution (dpi)")); + ui.add(egui::DragValue::new(&mut task.settings.dpi).range(72.0..=600.0)); + }); + ui.checkbox(&mut task.settings.recognize, t("Add searchable text")); + ui.checkbox(&mut task.settings.skip_text_pages, t("Skip pages that already contain text")); + ui.checkbox(&mut task.settings.auto_rotate, t("Automatically correct page orientation")); + ui.checkbox(&mut task.settings.deskew, t("Straighten skewed scans")); + ui.checkbox(&mut task.settings.auto_crop, t("Crop to detected paper edges")); + ui.add_enabled_ui(task.settings.auto_crop, |ui| { + ui.horizontal(|ui| { + ui.label(t("Safety margin (pt)")); + ui.add(egui::DragValue::new(&mut task.settings.crop_margin).range(0.0..=20.0)); + }); + }); + ui.weak(t("Uncertain orientations and paper edges are kept unchanged.")); + image_preparation(ui, &mut task.preprocessing, lang); + ui.checkbox(&mut task.remove_blank_pages, t("Remove completely white pages")); + if task.remove_blank_pages { + let mut archive = task.blank_archive.is_some(); + if ui.checkbox(&mut archive, t("Save removed blank pages separately")).changed() { + task.blank_archive = archive.then(PathBuf::new); + } + if let Some(path) = &mut task.blank_archive { + folder(ui, path, t("Blank page archive"), lang); + } + } + ui.weak(t("Faint marks and uncertain blank pages are preserved.")); + } + }); + ui.separator(); + if let Some(status) = self.state.tasks.get(&task.id) { + ui.label(format!( + "{}: {} {}: {} {}: {}", + t("Completed"), + status.completed, + t("Waiting"), + status.waiting, + t("Failed"), + status.failed + )); + if let Some(output) = &status.last_output { + ui.label(output.display().to_string()); + } + if let Some(error) = &status.error { + ui.collapsing(t("Error details"), |ui| { + ui.label(error); + }); + } + retry = ui.add_enabled(!task.enabled, egui::Button::new(t("Retry failed files"))).clicked(); + } + if ui.button(t("Open output folder")).clicked() { + self.error = open_folder(&task.output).err(); + } + ui.collapsing(t("Log"), |ui| { + for entry in self.state.log.iter().rev().filter(|e| e.task == task.id).take(100) { + let time = i64::try_from(entry.at) + .ok() + .and_then(|v| Local.timestamp_opt(v, 0).single()) + .map(|v| v.format("%Y-%m-%d %H:%M:%S").to_string()) + .unwrap_or_default(); + ui.label(format!("{time} — {}", entry.input.as_ref().map(|v| v.display().to_string()).unwrap_or_default())); + if let Some(error) = &entry.error { + ui.label(format!("{}: {error}", t("Error details"))); + } + if let Some(output) = &entry.output { + ui.label(output.display().to_string()); + } + ui.separator(); + } + }); + } else { + ui.label(t("Create a task to process new scans automatically.")); + } + if save { + self.error = self.save().err(); + } + if self.source_ui.has_pending() { + if let Err(error) = self.save() { + self.source_ui.cancel_pending(); + self.error = Some(error); + } else { + self.error = self.source_ui.begin_pending(&self.path).err(); + } + } + if start || scheduled { + self.error = self.start(start).err(); + } + if stop { + self.error = self.stop(false).err(); + } + if duplicate { + self.new_task(true); + } + if delete && self.selected < self.config.tasks.len() { + self.config.tasks.remove(self.selected); + let next = self.selected.min(self.config.tasks.len().saturating_sub(1)); + // The old row is gone; do not apply its date field to the following task. + self.selected = usize::MAX; + self.select(next); + self.error = self.save().err(); + } + if retry && let Some(task) = self.config.tasks.get(self.selected) { + self.error = hf::reset(&self.path, &task.id, None).err(); + } + if self.dirty() { + ui.weak(t("Unsaved changes")); + } else if self.saved { + ui.weak(t("Saved")); + } + }); + }); + } +} + +fn open_folder(path: &std::path::Path) -> Result<(), String> { + if !path.is_dir() { + return Err("output folder does not exist".into()); + } + #[cfg(windows)] + let program = "explorer.exe"; + #[cfg(target_os = "macos")] + let program = "open"; + #[cfg(not(any(windows, target_os = "macos")))] + let program = "xdg-open"; + std::process::Command::new(program).arg(path).spawn().map_err(|e| e.to_string())?; + Ok(()) +} + +fn folder(ui: &mut egui::Ui, value: &mut PathBuf, label: &str, language: Language) { + ui.label(label); + ui.horizontal(|ui| { + let mut text = value.to_string_lossy().to_string(); + if ui.add(egui::TextEdit::singleline(&mut text).desired_width((ui.available_width() - 110.0).max(100.0))).changed() { + *value = text.into(); + } + if ui.button(language.t("Browse…")).clicked() + && let Some(path) = rfd::FileDialog::new().set_title(label).pick_folder() + { + *value = path; + } + }); +} + +fn image_preparation(ui: &mut egui::Ui, settings: &mut pdfcraft_engine::hot_folder_preprocess::Settings, language: Language) { + let t = |key| language.t(key); + ui.collapsing(t("Page selection and image preparation"), |ui| { + ui.label(t("Pages to process (empty = all; example: 1-3,5)")); + ui.text_edit_singleline(&mut settings.pages); + ui.weak(t("Only selected pages are included in the output.")); + ui.checkbox(&mut settings.grayscale, t("Convert to grayscale")); + ui.checkbox(&mut settings.invert, t("Invert colors")); + ui.checkbox(&mut settings.median_filter, t("Reduce noise (median filter)")); + ui.checkbox(&mut settings.split_spreads, t("Split spreads into left and right pages")); + ui.horizontal(|ui| { + ui.label(t("Brightness")); + ui.add(egui::DragValue::new(&mut settings.brightness).range(-100..=100)); + ui.label(t("Contrast")); + ui.add(egui::DragValue::new(&mut settings.contrast).range(-100.0..=100.0)); + }); + let mut binary = settings.threshold.is_some(); + if ui.checkbox(&mut binary, t("Convert to black and white")).changed() { + settings.threshold = binary.then_some(128); + } + if let Some(threshold) = &mut settings.threshold { + ui.horizontal(|ui| { + ui.label(t("Threshold")); + ui.add(egui::DragValue::new(threshold).range(1..=254)); + }); + } + if settings.changes_images() { + ui.weak(t("Image changes rasterize pages at the chosen resolution. Documents with forms, links or annotations are refused.")); + } + }); + ui.collapsing(t("Recognition areas (saved with task)"), |ui| { + ui.weak(t("Coordinates are percentages from the top left. Page 0 applies to every prepared page. Visible content remains unchanged.")); + let mut remove = None; + for (index, region) in settings.regions.iter_mut().enumerate() { + ui.push_id(index, |ui| { + ui.horizontal_wrapped(|ui| { + ui.label(t("Page")); + ui.add(egui::DragValue::new(&mut region.page).range(0..=10000)); + ui.checkbox(&mut region.exclude, t("Exclude this area")); + if ui.button(t("Remove area")).clicked() { + remove = Some(index); + } + }); + ui.horizontal_wrapped(|ui| { + for (label, value) in ["Left (%)", "Top (%)", "Width (%)", "Height (%)"].into_iter().zip(region.rect.iter_mut()) { + let mut percent = *value * 100.0; + ui.label(t(label)); + if ui.add(egui::DragValue::new(&mut percent).range(0.0..=100.0).speed(0.5)).changed() { + *value = percent / 100.0; + } + } + }); + ui.separator(); + }); + } + if let Some(index) = remove { + settings.regions.remove(index); + } + if ui.add_enabled(settings.regions.len() < 64, egui::Button::new(t("Add recognition area"))).clicked() { + settings.regions.push(Default::default()); + } + }); +} + +fn needs_ocr(task: &hf::Task) -> bool { + !task.analysis_only && (task.settings.recognize || task.settings.auto_rotate || task.settings.deskew || task.settings.auto_crop) +} + +fn output_steps(ui: &mut egui::Ui, plan: &mut output::OutputPlan, language: Language, analysis: bool) { + let t = |key| language.t(key); + ui.collapsing(t("Output saving steps"), |ui| { + ui.weak(t("All outputs must be saved before originals are moved or deleted. Existing files are never overwritten.")); + let count = plan.steps.len(); + let mut remove = None; + for (index, step) in plan.steps.iter_mut().enumerate() { + ui.push_id(index, |ui| { + ui.horizontal_wrapped(|ui| { + ui.label(format!("{} {}", t("Save"), index.saturating_add(1))); + ui.add_enabled_ui(!analysis, |ui| { + let shown = if analysis { output::Format::Analysis } else { step.format }; + egui::ComboBox::from_id_salt("format").selected_text(t(shown.label())).show_ui(ui, |ui| { + for format in output::Format::ALL { + ui.selectable_value(&mut step.format, *format, t(format.label())); + } + }); + }); + if ui.add_enabled(count > 1, egui::Button::new(t("Remove step"))).clicked() { + remove = Some(index); + } + }); + egui::ComboBox::from_id_salt("group").selected_text(t(grouping_label(step.grouping))).show_ui(ui, |ui| { + for grouping in [output::Grouping::File, output::Grouping::Folder, output::Grouping::All] { + ui.selectable_value(&mut step.grouping, grouping, t(grouping_label(grouping))); + } + }); + let mut custom = step.folder.is_some(); + if ui.checkbox(&mut custom, t("Use another output folder")).changed() { + step.folder = custom.then(PathBuf::new); + } + if let Some(path) = &mut step.folder { + folder(ui, path, t("Output folder"), language); + } + ui.label(t("Filename template (without extension)")); + ui.text_edit_singleline(&mut step.name_template); + ui.weak("{name} · {folder} · {task} · {date} · {time} · {counter}"); + if !analysis && matches!(step.format, output::Format::Png | output::Format::Jpeg | output::Format::Tiff | output::Format::ImagePdf) { + ui.horizontal(|ui| { + ui.label(t("Resolution (dpi)")); + ui.add(egui::DragValue::new(&mut step.dpi).range(18.0..=1200.0)); + }); + } + if !analysis && step.format == output::Format::Jpeg { + ui.horizontal(|ui| { + ui.label(t("JPEG quality")); + ui.add(egui::DragValue::new(&mut step.jpeg_quality).range(1..=100)); + }); + } + ui.separator(); + }); + } + if let Some(index) = remove { + plan.steps.remove(index); + } + if ui.add_enabled(plan.steps.len() < 16, egui::Button::new(t("Add saving step"))).clicked() { + plan.steps.push(output::SaveStep::default()); + } + }); +} + +fn grouping_label(group: output::Grouping) -> &'static str { + match group { + output::Grouping::File => "One document per file", + output::Grouping::Folder => "Combine files in each folder", + output::Grouping::All => "Combine all files in this run", + } +} + +fn calendar(ui: &mut egui::Ui, schedule: &mut hf::Schedule, language: Language) { + let t = |key| language.t(key); + let mut kind = match schedule { + hf::Schedule::Interval => 0, + hf::Schedule::Daily { .. } => 1, + hf::Schedule::Weekly { .. } => 2, + hf::Schedule::Monthly { .. } => 3, + }; + let old = kind; + let names = ["At regular intervals", "Daily", "Weekly", "Monthly"]; + egui::ComboBox::from_id_salt("recurrence").selected_text(t(names.get(kind).copied().unwrap_or(names[0]))).show_ui(ui, |ui| { + for (index, name) in names.iter().enumerate() { + ui.selectable_value(&mut kind, index, t(name)); + } + }); + if old != kind { + *schedule = match kind { + 1 => hf::Schedule::Daily { hour: 8, minute: 0 }, + 2 => hf::Schedule::Weekly { weekday: 0, hour: 8, minute: 0 }, + 3 => hf::Schedule::Monthly { day: 1, hour: 8, minute: 0 }, + _ => hf::Schedule::Interval, + }; + } + if let hf::Schedule::Weekly { weekday, .. } = schedule { + let weekdays = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]; + egui::ComboBox::from_id_salt("weekday").selected_text(t(weekdays.get(*weekday as usize).copied().unwrap_or(weekdays[0]))).show_ui(ui, |ui| { + for (index, name) in weekdays.iter().enumerate() { + ui.selectable_value(weekday, index as u32, t(name)); + } + }); + } + if let hf::Schedule::Monthly { day, .. } = schedule { + ui.horizontal(|ui| { + ui.label(t("Day of month")); + ui.add(egui::DragValue::new(day).range(1..=31)); + }); + } + match schedule { + hf::Schedule::Daily { hour, minute } | hf::Schedule::Weekly { hour, minute, .. } | hf::Schedule::Monthly { hour, minute, .. } => { + ui.horizontal(|ui| { + ui.label(t("Time")); + ui.add(egui::DragValue::new(hour).range(0..=23)); + ui.label(":"); + ui.add(egui::DragValue::new(minute).range(0..=59)); + }); + } + hf::Schedule::Interval => {} + } +} + +fn read_disk(path: &std::path::Path) -> Result>, String> { + use std::io::Read; + match std::fs::File::open(path) { + Ok(file) => { + let mut bytes = Vec::new(); + file.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes).map_err(|e| e.to_string())?; + if bytes.len() > 16 * 1024 * 1024 { + return Err("configuration exceeds 16 MiB".into()); + } + Ok(Some(bytes)) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.to_string()), + } +} + +impl eframe::App for HotFolderApp { + fn ui(&mut self, ui: &mut egui::Ui, _frame: &mut eframe::Frame) { + let ctx = ui.ctx().clone(); + if !self.allow_close && ctx.input(|i| i.viewport().close_requested()) && self.dirty() { + ctx.send_viewport_cmd(egui::ViewportCommand::CancelClose); + self.close_prompt = true; + } + self.show(ui); + if self.close_prompt { + let lang = self.language; + egui::Modal::new(egui::Id::new("unsaved-hot-folder")).show(&ctx, |ui| { + ui.heading(lang.t("Unsaved changes")); + ui.label(lang.t("Save your task settings before closing?")); + ui.horizontal(|ui| { + if ui.button(lang.t("Save and close")).clicked() { + match self.save() { + Ok(()) => { + self.allow_close = true; + ctx.send_viewport_cmd(egui::ViewportCommand::Close); + } + Err(error) => self.error = Some(error), + } + self.close_prompt = false; + } + if ui.button(lang.t("Discard and close")).clicked() { + self.allow_close = true; + ctx.send_viewport_cmd(egui::ViewportCommand::Close); + } + if ui.button(lang.t("Cancel")).clicked() { + self.close_prompt = false; + } + }); + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use egui_kittest::{Harness, kittest::Queryable}; + + #[test] + fn hot_folder_small_window_keeps_actions_tasks_service_and_errors_reachable_in_every_language() { + for (language, (code, _)) in LANGUAGES.iter().enumerate() { + let dir = tempfile::tempdir().unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(780.0, 560.0)).build_eframe(|cc| { + install_fonts(&cc.egui_ctx); + let mut app = HotFolderApp::new(dir.path().join("tasks.json")); + app.language = Language(language); + app.inline = true; + app.error = None; + for index in 0..32 { + app.new_task(false); + app.config.tasks[index].name = format!("Task {}", index + 1); + app.config.tasks[index].analysis_only = true; + } + app.select(0); + app + }); + h.run_steps(10); + let viewport = egui::Rect::from_min_size(egui::Pos2::ZERO, egui::vec2(780.0, 560.0)); + let t = |key| Language(language).t(key); + for key in ["Save", "Start now", "Start on schedule", "Stop", "Duplicate", "Delete task"] { + assert!(viewport.contains_rect(h.get_by_label(t(key)).rect()), "{code}: {key} must fit the minimum window"); + } + // The long task list must scroll to both its final row and the service controls. + let last = format!("Task 32\n{}", t("Stopped")); + h.get_by_label(&last).scroll_to_me(); + h.run_steps(10); + assert!(viewport.contains_rect(h.get_by_label(&last).rect()), "{code}: final task must be reachable"); + h.get_by_label(t("Stop all")).scroll_to_me(); + h.run_steps(10); + assert!(viewport.contains_rect(h.get_by_label(t("Stop all")).rect()), "{code}: Stop all must be reachable"); + h.get_by_label(t("System service")).scroll_to_me(); + h.run_steps(10); + h.get_by_label(t("System service")).click(); + h.run_steps(10); + h.get_by_label(t("Refresh status")).scroll_to_me(); + h.run_steps(10); + assert!(viewport.contains_rect(h.get_by_label(t("Refresh status")).rect()), "{code}: service controls must be reachable"); + // An invalid input path must report the start error at the top, without a scroll. + h.get_by_label(t("Start now")).click(); + h.run_steps(10); + let error = h.state().error.clone().expect("unspecified task folders must fail validation"); + assert!(viewport.contains_rect(h.get_by_label(t("Error details")).rect()), "{code}: error heading must stay visible"); + assert!(viewport.intersects(h.get_by_label(&error).rect()), "{code}: failure detail must stay visible"); + if let Ok(shots) = std::env::var("PDFCRAFT_HOT_FOLDER_SMALL_SHOTS") { + h.render().unwrap().save(std::path::Path::new(&shots).join(format!("hot-folder-{code}-small.png"))).unwrap(); + } + let input = dir.path().join("in"); + let output = dir.path().join("out"); + std::fs::create_dir(&input).unwrap(); + std::fs::create_dir(&output).unwrap(); + for task in &mut h.state_mut().config.tasks { + task.input = input.clone(); + task.output = output.clone(); + } + h.get_by_label(t("Start now")).click(); + h.run_steps(10); + assert!(h.state().error.is_none(), "{code}: {:?}", h.state().error); + assert!(hf::load_config(&h.state().path).unwrap().tasks[0].enabled, "{code}: start must enable the selected task"); + h.get_by_label(t("Stop")).click(); + h.run_steps(10); + assert!(h.state().error.is_none(), "{code}: {:?}", h.state().error); + assert!(!hf::load_config(&h.state().path).unwrap().tasks[0].enabled, "{code}: stop must persist the cancellation"); + } + } + + #[test] + fn hot_folder_ui_preserves_settings_changed_by_another_window() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("tasks.json"); + let mut app = HotFolderApp::new(path.clone()); + app.language = Language(0); + app.new_task(false); + for name in ["in", "out"] { + std::fs::create_dir(dir.path().join(name)).unwrap(); + } + app.config.tasks[0].input = dir.path().join("in"); + app.config.tasks[0].output = dir.path().join("out"); + assert!(app.dirty()); + hf::save_config(&path, &hf::Config::default(), None).unwrap(); + assert!(app.save().unwrap_err().contains("Reload")); + assert!(hf::load_config(&path).unwrap().tasks.is_empty()); + } + + #[test] + fn hot_folder_ui_has_seven_languages_and_persists_a_stopped_task() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("in"); + let output = dir.path().join("out"); + std::fs::create_dir_all(&input).unwrap(); + std::fs::create_dir_all(&output).unwrap(); + let path = dir.path().join("tasks.json"); + let mut h = Harness::builder().with_size(egui::vec2(1100.0, 1000.0)).build_eframe(|cc| { + install_fonts(&cc.egui_ctx); + let mut app = HotFolderApp::new(path.clone()); + app.language = Language(0); + app.inline = true; + app + }); + h.run_steps(2); + h.get_by_label("New task").click(); + h.run_steps(2); + let task = h.state_mut().config.tasks.first_mut().unwrap(); + task.input = input; + task.output = output; + h.get_by_label("Save").click(); + h.run_steps(2); + let config = hf::load_config(&path).unwrap(); + assert_eq!(config.tasks.len(), 1); + assert!(!config.tasks[0].enabled); + assert_eq!(config.tasks[0].originals, hf::Originals::Keep); + use pdfcraft_engine::hot_folder_sources::SourceConfig; + for (source, label) in [ + (SourceConfig::Ftp(Default::default()), "Save password securely"), + (SourceConfig::Outlook(Default::default()), "Sign in to Outlook"), + (SourceConfig::Project(Default::default()), "Choose project…"), + ] { + h.state_mut().config.tasks[0].source = source; + h.run_steps(2); + h.get_by_label(label); + assert!(!h.state().source_ui.has_pending()); + } + h.state_mut().config.tasks[0].source = SourceConfig::Local; + for index in 0..LANGUAGES.len() { + h.state_mut().language = Language(index); + h.run_steps(2); + h.get_by_label(Language(index).t("Task name")); + } + if let Ok(path) = std::env::var("PDFCRAFT_HOT_FOLDER_SHOT") { + h.state_mut().language = Language(1); + h.run_steps(3); + h.render().unwrap().save(path).unwrap(); + } + } +} diff --git a/apps/pdfcraft-hot-folder/src/worker.rs b/apps/pdfcraft-hot-folder/src/worker.rs new file mode 100644 index 0000000000..722559055c --- /dev/null +++ b/apps/pdfcraft-hot-folder/src/worker.rs @@ -0,0 +1,61 @@ +//! Background processing loop used by the manual worker and the system-service host. +use pdfcraft_engine::hot_folder as hf; +use std::path::Path; + +pub fn check_identity() -> Result<(), String> { + #[cfg(unix)] + if rustix::process::geteuid().is_root() { + return Err("Run folder processing as the configured non-root service account; administrator mode is only for setup".into()); + } + Ok(()) +} + +pub fn run(config: &Path, should_stop: impl Fn() -> bool) -> Result<(), String> { + check_identity()?; + if !config.exists() { + hf::initialize_config(config)?; + } + let lock = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(config.with_extension("worker.lock")) + .map_err(|e| e.to_string())?; + if lock.try_lock().is_err() { + return Ok(()); + } + while !should_stop() { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hf::poll_cancelable(config, None, &should_stop))) + .unwrap_or_else(|_| Err("processing failed safely; source files were retained".into())); + match result { + Ok(_) => { + hf::record_worker_error(config, None)?; + } + Err(error) => { + hf::record_worker_error(config, Some(&error))?; + } + } + if !should_stop() { + std::thread::sleep(std::time::Duration::from_secs(1)); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn hot_folder_worker_initializes_missing_stopped_configuration_as_its_account() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nested/tasks.json"); + #[cfg(unix)] + if rustix::process::geteuid().is_root() { + assert!(super::run(&path, || true).unwrap_err().contains("non-root")); + assert!(!path.exists()); + return; + } + super::run(&path, || true).unwrap(); + assert!(pdfcraft_engine::hot_folder::load_config(&path).unwrap().tasks.is_empty()); + } +} diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index 3f8d63e25b..c554d69d66 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -23,8 +23,15 @@ default = ["mcp"] # The MCP server module. Opt-in at run time either way; disable to leave it out of the build. mcp = ["dep:base64"] +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +pdfcraft-hot-folder-service = { path = "../hot-folder-service" } +zeroize = "1.8" + [dev-dependencies] pdfcraft-cos = { workspace = true } pdfcraft-model = { workspace = true } pdfcraft-xfa = { workspace = true } image = { version = "0.25", default-features = false, features = ["jpeg"] } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +tempfile = "3" diff --git a/crates/automation/README.md b/crates/automation/README.md index 02c0811495..219d462283 100644 --- a/crates/automation/README.md +++ b/crates/automation/README.md @@ -68,6 +68,18 @@ moves 1–1000 references atomically in displayed points (right/down), preservin and fonts. A refusal changes nothing; success is one undo step. See [the selection and movement guide](../../docs/editing-multiple-objects.md) for limits and examples. +## Hot Folder + +The native `hot_folder_configure`, `hot_folder_status`, `hot_folder_poll`, `hot_folder_retry` +and `hot_folder_run_now` tools share the standalone application's durable engine. Configuration +contains named tasks, existing disjoint folders, OCR options and interval or calendar schedules. +All nested paths and sidecars honour `--root`. Poll does one bounded scheduler pass; no tool +implicitly starts a service or background process. `hot_folder_project` writes ordered open +project manifests; `hot_folder_source` provides explicit remote authentication and folder discovery. +FTP passwords enter through named environment variables, never through replayable tool arguments. +`hot_folder_service` explicitly controls an installed boot service and refuses mutations under +`--root`. See [the configuration and scope guide](../../docs/hot-folder.md). + ## Common command tools Following FilmCraft's MCP conventions, `command_list {doc?, filter?, enabled_only?}` keeps its diff --git a/crates/automation/src/hot_folder.rs b/crates/automation/src/hot_folder.rs new file mode 100644 index 0000000000..a28df0989b --- /dev/null +++ b/crates/automation/src/hot_folder.rs @@ -0,0 +1,150 @@ +use crate::{Args, Automation, Result, failed}; +use serde_json::Value; +#[cfg(not(target_arch = "wasm32"))] +use serde_json::json; + +impl Automation { + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn hot_folder_project(&self, args: &Args) -> Result { + use pdfcraft_engine::hot_folder_sources as source; + let path = self.resolve(args.str("path")?, true)?; + let parent = path.parent().ok_or_else(|| failed("project needs a parent folder"))?; + let files = args + .strs("files")? + .into_iter() + .map(|file| { + let resolved = self.resolve(file, false)?; + resolved + .strip_prefix(parent) + .map(std::path::Path::to_path_buf) + .map_err(|_| failed("save the project in a folder containing all source files")) + }) + .collect::>>()?; + let manifest = source::ProjectManifest { files, ..Default::default() }; + source::write_project_manifest(&path, &manifest, self.root.as_deref()).map_err(failed)?; + Ok(json!({"path":path,"project":manifest})) + } + + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn hot_folder_source(&self, args: &Args) -> Result { + use pdfcraft_engine::{hot_folder as hf, hot_folder_sources as source}; + let path = self.resolve(args.str("config")?, false)?; + let config = hf::load_config(&path).map_err(failed)?; + let task_id = args.str("task")?; + let task = config.tasks.iter().find(|task| task.id == task_id).ok_or_else(|| failed("unknown task"))?; + let id = match &task.source { + source::SourceConfig::Ftp(settings) => &settings.credential_id, + source::SourceConfig::Outlook(settings) => &settings.credential_id, + _ => return Err(failed("this task has no authenticated remote source")), + }; + let action = args.str("action")?; + if task.enabled && matches!(action, "save_ftp_password" | "forget_credentials" | "outlook_begin" | "outlook_poll") { + return Err(failed("stop the task before changing its credentials")); + } + let store = source::CredentialStore::for_config(&path, self.root.as_deref()).map_err(failed)?; + match action { + "credential_status" => Ok(json!({"available":source::credential_status(&store,id).map_err(failed)?})), + "forget_credentials" => { + source::forget_credentials(&store, id).map_err(failed)?; + Ok(json!({"removed":true})) + } + "save_ftp_password" => { + if !matches!(task.source, source::SourceConfig::Ftp(_)) { + return Err(failed("passwords apply only to FTP tasks")); + } + // Tools accept a variable name, never a secret in replayable task arguments. + let variable = args.str("password_env")?; + if variable.is_empty() || variable.len() > 128 || !variable.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return Err(failed("password_env must name an environment variable")); + } + let password = + zeroize::Zeroizing::new(std::env::var(variable).map_err(|_| failed("the named password environment variable is unavailable"))?); + source::save_ftp_password(&store, id, &password).map_err(failed)?; + Ok(json!({"saved":true})) + } + "outlook_begin" | "outlook_poll" | "outlook_folders" => { + let source::SourceConfig::Outlook(settings) = &task.source else { + return Err(failed("this task is not an Outlook source")); + }; + match action { + "outlook_begin" => serde_json::to_value(source::outlook_begin_sign_in(&store, settings).map_err(failed)?).map_err(failed), + "outlook_poll" => { + let flow: source::DeviceSignIn = + serde_json::from_value(args.0.get("flow").cloned().ok_or_else(|| failed("flow is required"))?).map_err(failed)?; + serde_json::to_value(source::outlook_poll_sign_in(&store, settings, &flow).map_err(failed)?).map_err(failed) + } + _ => serde_json::to_value(source::outlook_folders(&store, settings, args.opt_str("parent")?, &|| false).map_err(failed)?) + .map_err(failed), + } + } + _ => Err(failed("unknown source action")), + } + } + + #[cfg(target_arch = "wasm32")] + pub(crate) fn hot_folder_source(&self, _args: &Args) -> Result { + Err(failed("Hot Folder sources require native CLI or desktop")) + } + + #[cfg(target_arch = "wasm32")] + pub(crate) fn hot_folder_project(&self, _args: &Args) -> Result { + Err(failed("Hot Folder projects require native CLI or desktop")) + } + + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn hot_folder(&self, action: &str, args: &Args) -> Result { + use pdfcraft_engine::hot_folder as hf; + let path = self.resolve(args.str("config")?, action == "hot_folder_configure")?; + // Validate every nested path with the same root resolver as other automation tools. + let mut config: hf::Config = if action == "hot_folder_configure" { + serde_json::from_value(args.0.get("settings").cloned().unwrap_or(json!({}))).map_err(failed)? + } else { + hf::load_config(&path).map_err(failed)? + }; + for t in &mut config.tasks { + t.input = self.resolve(&t.input.to_string_lossy(), false)?; + t.output = self.resolve(&t.output.to_string_lossy(), false)?; + if let pdfcraft_engine::hot_folder_sources::SourceConfig::Project(source) = &mut t.source { + source.manifest = self.resolve(&source.manifest.to_string_lossy(), false)?; + if let pdfcraft_engine::hot_folder_sources::SourceAction::Move { destination } = &mut source.action { + *destination = self.resolve(destination, false)?.to_string_lossy().into_owned(); + } + } + if let Some(folder) = &mut t.blank_archive { + *folder = self.resolve(&folder.to_string_lossy(), false)?; + } + for step in &mut t.output_plan.steps { + if let Some(folder) = &mut step.folder { + *folder = self.resolve(&folder.to_string_lossy(), false)?; + } + } + if t.originals == hf::Originals::Move { + t.archive = self.resolve(&t.archive.to_string_lossy(), false)?; + } + } + hf::validate(&config, self.root.as_deref()).map_err(failed)?; + self.resolve(&hf::state_path(&path).to_string_lossy(), true)?; + self.resolve(&path.with_extension("lock").to_string_lossy(), true)?; + match action { + "hot_folder_configure" => { + hf::save_config(&path, &config, self.root.as_deref()).map_err(failed)?; + Ok(json!({ "config": path, "settings": config })) + } + "hot_folder_poll" => serde_json::to_value(hf::poll(&path, self.root.as_deref()).map_err(failed)?).map_err(failed), + "hot_folder_retry" => { + hf::reset(&path, args.str("task")?, self.root.as_deref()).map_err(failed)?; + Ok(json!({ "reset": true })) + } + "hot_folder_run_now" => { + hf::run_now(&path, args.str("task")?, self.root.as_deref()).map_err(failed)?; + Ok(json!({ "scheduled_now": true })) + } + _ => Ok(json!({ "settings": config, "state": hf::read_state(&path).map_err(failed)? })), + } + } + + #[cfg(target_arch = "wasm32")] + pub(crate) fn hot_folder(&self, _action: &str, _args: &Args) -> Result { + Err(failed("Hot Folder requires the desktop application or native CLI")) + } +} diff --git a/crates/automation/src/hot_folder_service.rs b/crates/automation/src/hot_folder_service.rs new file mode 100644 index 0000000000..87e9eb5b27 --- /dev/null +++ b/crates/automation/src/hot_folder_service.rs @@ -0,0 +1,51 @@ +use crate::{Args, Automation, Result, failed}; +use serde_json::Value; + +impl Automation { + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn hot_folder_service(&self, args: &Args) -> Result { + use pdfcraft_hot_folder_service as service; + let action = args.str("action")?; + // A filesystem root cannot confine system service changes. Never silently cross it. + if action != "status" && self.root.is_some() { + return Err(failed( + "System service changes are unavailable in a --root confined session; run an explicit unrestricted administration command", + )); + } + let executable = self.resolve(args.str("executable")?, false)?; + let config = self.resolve(args.str("config")?, false)?; + match action { + "status" => {} + "start" => service::start(&executable, &config).map_err(failed)?, + "stop" => service::stop(&executable, &config).map_err(failed)?, + "enable" => service::set_enabled(true, &executable, &config).map_err(failed)?, + "disable" => service::set_enabled(false, &executable, &config).map_err(failed)?, + "install" => service::install(&executable, &config, args.str("account")?).map_err(failed)?, + "uninstall" => service::uninstall(&executable, &config).map_err(failed)?, + _ => return Err(failed("action must be status, start, stop, enable, disable, install or uninstall")), + } + serde_json::to_value(service::status(&executable, &config).map_err(failed)?).map_err(failed) + } + + #[cfg(target_arch = "wasm32")] + pub(crate) fn hot_folder_service(&self, _: &Args) -> Result { + Err(failed("System services require the native application or CLI")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn system_changes_never_escape_a_confined_automation_session() { + let root = std::env::temp_dir(); + let mut automation = Automation::new(); + automation.root = Some(root); + for action in ["start", "stop", "enable", "disable", "install", "uninstall"] { + let value = json!({"action":action,"executable":"unneeded","config":"unneeded","account":"root"}); + assert!(automation.hot_folder_service(&Args(&value)).unwrap_err().to_string().contains("confined")); + } + } +} diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index b856e978ef..ba3c5383f6 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -16,6 +16,8 @@ mod comments; mod content; mod conventions; mod forms; +mod hot_folder; +mod hot_folder_service; mod links; #[cfg(feature = "mcp")] pub mod mcp; @@ -589,6 +591,12 @@ impl Automation { "doc_compare_mark" => self.doc_compare_mark(&a)?, "form_set_actions" => self.form_set_actions(&a)?, "ocr_status" => self.ocr_status()?, + "hot_folder_service" => self.hot_folder_service(&a)?, + "hot_folder_source" => self.hot_folder_source(&a)?, + "hot_folder_project" => self.hot_folder_project(&a)?, + "hot_folder_configure" | "hot_folder_poll" | "hot_folder_status" | "hot_folder_retry" | "hot_folder_run_now" => { + self.hot_folder(name, &a)? + } "ocr_recognize_files" => self.ocr_recognize_files(&a)?, "accessibility_report" => self.a11y_report(&a)?, "accessibility_fix" => self.a11y_fix(&a)?, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index d1ae2e19c2..b21234a1a7 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -952,6 +952,23 @@ pub fn tools() -> Vec { .ro() .cmd("ocr.recognize") .with(schema(json!({}), &[])), + t("hot_folder_configure", "Configure Hot Folder", "Save the standalone Hot Folder configuration atomically. Does not launch a background worker. Tasks contain id, name, existing disjoint input/output folders, enabled (default false), recursive, pdf, images, start_at (Unix seconds), once, interval_seconds (1..86400), settle_seconds (1..300), originals (keep/move/delete, default keep), archive (for move), and OCR settings. All nested paths honour --root. Set enabled false to stop a task; partial processing is discarded. See docs/hot-folder.md.") + .destructive() + .with(schema(json!({"config":{"type":"string"}, "settings":{"type":"object", "properties":{"tasks":{"type":"array", "maxItems":32, "items":{"type":"object"}}}, "required":["tasks"], "additionalProperties":false}}), &["config", "settings"])), + t("hot_folder_poll", "Process ready Hot Folder inputs", "Run one scheduler pass (one stable input or combined batch per enabled task), saving durable status and receipts. Call again after the configured interval. Existing outputs are never overwritten. Explicit originals move/delete policies act only after all output steps and receipts succeed. Does not start a background worker.") + .destructive().with(schema(json!({"config":{"type":"string"}}), &["config"])), + t("hot_folder_status", "Hot Folder status", "Read configuration and durable task status, counts, last input/output and actionable errors. No background work is started.") + .ro().with(schema(json!({"config":{"type":"string"}}), &["config"])), + t("hot_folder_retry", "Retry a Hot Folder task", "Clear failed attempts and reschedule a completed once task. Successful output receipts are preserved to prevent duplicate processing. Stop the task first if a worker is busy.") + .with(schema(json!({"config":{"type":"string"}, "task":{"type":"string"}}), &["config", "task"])), + t("hot_folder_run_now", "Run Hot Folder at the next poll", "Make an enabled task due at its next poll, overriding its calendar schedule once. Successful receipts remain. The task must already be enabled and its start_at must be due; no worker is started.") + .with(schema(json!({"config":{"type":"string"}, "task":{"type":"string"}}), &["config", "task"])), + t("hot_folder_service", "Manage Hot Folder system service", "Inspect or explicitly manage the system boot service matching executable and config. Changes require platform administrator rights and are unavailable under --root. Unix install requires a non-root account; Windows installation is performed by the optional MSI component. Nothing is installed implicitly.") + .destructive().with(schema(json!({"executable":{"type":"string"},"config":{"type":"string"},"action":{"type":"string","enum":["status","start","stop","enable","disable","install","uninstall"]},"account":{"type":"string"}}), &["executable","config","action"])), + t("hot_folder_source", "Connect Hot Folder source", "Manage the configured task's protected credentials or explicit Outlook device sign-in. No passwords or access tokens are returned. save_ftp_password reads a named environment variable; flow is the public object returned by outlook_begin. Stop the task before changing credentials. Credentials require the installer-provisioned service store. Outlook needs your own public-client application registration. No work starts implicitly.") + .destructive().with(schema(json!({"config":{"type":"string"},"task":{"type":"string"},"action":{"type":"string","enum":["credential_status","save_ftp_password","forget_credentials","outlook_begin","outlook_poll","outlook_folders"]},"password_env":{"type":"string"},"flow":{"type":"object"},"parent":{"type":"string"}}), &["config","task","action"])), + t("hot_folder_project", "Create Hot Folder project", "Create an open PDFCraft manifest containing ordered PDF/image paths. This is not an ABBYY proprietary project. All files must be inside the manifest's parent folder. Existing manifests are never overwritten; all paths honour --root.") + .with(schema(json!({"path":{"type":"string"},"files":{"type":"array","minItems":1,"maxItems":10000,"items":{"type":"string"}}}), &["path","files"])), t("doc_export_all_images", "Export all images", "Write the images that pages use into a folder (`_Page__Image_.jpg|png`), each once: JPEG images unchanged, others as PNG with their soft mask as alpha. min_size skips images with fewer pixels on their shorter side. Images that can't be decoded yet (JPEG 2000, JBIG2, CCITT, separations) are listed under skipped. Includes unsaved edits.") .cmd("export.all_images") .with(schema( diff --git a/crates/automation/tests/hot_folder.rs b/crates/automation/tests/hot_folder.rs new file mode 100644 index 0000000000..1a919df83c --- /dev/null +++ b/crates/automation/tests/hot_folder.rs @@ -0,0 +1,163 @@ +#![cfg(not(target_arch = "wasm32"))] +use pdfcraft_automation::{Automation, Content}; +use serde_json::{Value, json}; + +fn call(a: &mut Automation, name: &str, args: Value) -> Value { + match a.call(name, &args).unwrap().remove(0) { + Content::Json(value) => value, + _ => panic!("expected JSON"), + } +} + +#[test] +fn hot_folder_analysis_only_preserves_pages_and_skips_processing_options() { + let root = tempfile::tempdir().unwrap(); + for folder in ["in", "out"] { + std::fs::create_dir(root.path().join(folder)).unwrap(); + } + let session = pdfcraft_engine::Session::new(); + let text = session.create_from_text("source", "Analyze existing words without changing them").unwrap(); + let blank = session.create_blank(100.0, 100.0, 1).unwrap(); + let source = session.combine(&[("text.pdf".into(), text), ("blank.pdf".into(), blank)]).unwrap(); + std::fs::write(root.path().join("in/source.pdf"), source.as_slice()).unwrap(); + let mut a = Automation::new().with_root(root.path()).unwrap(); + let task = json!({"id":"analysis","name":"Analysis","input":"in","output":"out","analysis_only":true,"remove_blank_pages":true,"enabled":true,"interval_seconds":1,"settle_seconds":1,"preprocessing":{"invert":true,"split_spreads":true},"output_plan":{"steps":[{"format":"pdf"}]}}); + call(&mut a, "hot_folder_configure", json!({"config":"tasks.json","settings":{"tasks":[task]}})); + call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + std::thread::sleep(std::time::Duration::from_millis(1100)); + let state = call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + assert_eq!(state["tasks"]["analysis"]["completed"], 1, "{state}"); + assert_eq!(std::fs::read_dir(root.path().join("out")).unwrap().count(), 1); + let report: Value = serde_json::from_slice(&std::fs::read(root.path().join("out/source.json")).unwrap()).unwrap(); + assert_eq!(report["format"], "pdfcraft-analysis"); + assert_eq!(report["ocr_performed"], false); + assert_eq!(report["pages"].as_array().unwrap().len(), 2); + assert_eq!(report["pages"][0]["content"], "existing_text"); + assert_eq!(report["pages"][1]["content"], "empty_content"); + assert_eq!(std::fs::read(root.path().join("in/source.pdf")).unwrap(), *source); + let config = call(&mut a, "hot_folder_status", json!({"config":"tasks.json"})); + assert_eq!(config["settings"]["tasks"][0]["settings"]["recognize"], true); + assert_eq!(config["settings"]["tasks"][0]["output_plan"]["steps"][0]["format"], "pdf"); +} + +#[test] +fn hot_folder_project_sources_preserve_order_and_ack_only_after_all_outputs() { + let root = std::env::temp_dir().join(format!("pdfcraft-hot-folder-project-tools-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + for folder in ["sources", "stage", "out"] { + std::fs::create_dir_all(root.join(folder)).unwrap(); + } + let session = pdfcraft_engine::Session::new(); + for (name, text) in [("z.pdf", "Zebra comes first"), ("a.pdf", "Apple comes second")] { + std::fs::write(root.join("sources").join(name), session.create_from_text("source", text).unwrap().as_slice()).unwrap(); + } + let mut a = Automation::new().with_root(&root).unwrap(); + let project = call(&mut a, "hot_folder_project", json!({"path":"sources/project.json","files":["sources/z.pdf","sources/a.pdf"]})); + assert_eq!(project["project"]["files"], json!(["z.pdf", "a.pdf"])); + assert!(a.call("hot_folder_project", &json!({"path":"../escaped.json","files":["sources/z.pdf"]})).is_err()); + assert!(a.call("hot_folder_project", &json!({"path":"sources/project.json","files":["sources/z.pdf"]})).is_err()); + let task = json!({"id":"project","name":"Project","input":"stage","output":"out","enabled":true,"interval_seconds":1,"settle_seconds":1,"settings":{"language":"de","recognize":false,"auto_rotate":false,"deskew":false},"source":{"kind":"project","manifest":"sources/project.json","action":{"kind":"delete"}},"output_plan":{"steps":[{"format":"pdf","grouping":"all","name_template":"combined"},{"format":"txt","grouping":"all","name_template":"combined"},{"format":"xlsx","grouping":"all","name_template":"combined"},{"format":"epub","grouping":"all","name_template":"combined"}]}}); + call(&mut a, "hot_folder_configure", json!({"config":"tasks.json","settings":{"tasks":[task]}})); + let mut overlapping = call(&mut a, "hot_folder_status", json!({"config":"tasks.json"}))["settings"].clone(); + overlapping["tasks"].as_array_mut().unwrap().push(json!({"id":"other","name":"Other","input":"sources","output":"out","enabled":true})); + assert!(a.call("hot_folder_configure", &json!({"config":"overlap.json","settings":overlapping})).unwrap_err().to_string().contains("overlap")); + call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + assert!(root.join("sources/z.pdf").exists()); + std::thread::sleep(std::time::Duration::from_millis(1100)); + let state = call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + assert_eq!(state["tasks"]["project"]["completed"], 2, "{state}"); + assert!(!root.join("sources/z.pdf").exists()); + assert!(!root.join("sources/a.pdf").exists()); + let text = std::fs::read_to_string(root.join("out/combined.txt")).unwrap(); + assert!(text.find("Zebra").unwrap() < text.find("Apple").unwrap(), "{text}"); + call(&mut a, "hot_folder_retry", json!({"config":"tasks.json","task":"project"})); + call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + assert_eq!(std::fs::read_dir(root.join("out")).unwrap().count(), 4); + for extension in ["xlsx", "epub"] { + assert!(std::fs::read(root.join(format!("out/combined.{extension}"))).unwrap().starts_with(b"PK\x03\x04")); + } + let error = a.call("hot_folder_source", &json!({"config":"tasks.json","task":"project","action":"credential_status"})).unwrap_err(); + assert!(error.to_string().contains("no authenticated")); +} + +#[test] +fn hot_folder_tools_configure_poll_stop_and_confine_all_nested_paths() { + let root = std::env::temp_dir().join(format!("pdfcraft-hot-folder-tools-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("in")).unwrap(); + std::fs::create_dir_all(root.join("out")).unwrap(); + let mut a = Automation::new().with_root(&root).unwrap(); + let task = + json!({"id":"incoming", "name":"Incoming scans", "input":"in", "output":"out", "enabled":true, "settle_seconds":1, "interval_seconds":1}); + call(&mut a, "hot_folder_configure", json!({"config":"tasks.json","settings":{"tasks":[task.clone()]}})); + let state = call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + assert_eq!(state["tasks"]["incoming"]["completed"], 0); + let status = call(&mut a, "hot_folder_status", json!({"config":"tasks.json"})); + assert_eq!(status["settings"]["tasks"][0]["enabled"], true); + let mut stopped = task.clone(); + stopped["enabled"] = json!(false); + call(&mut a, "hot_folder_configure", json!({"config":"tasks.json","settings":{"tasks":[stopped]}})); + call(&mut a, "hot_folder_retry", json!({"config":"tasks.json","task":"incoming"})); + assert!(call(&mut a, "hot_folder_status", json!({"config":"tasks.json"}))["state"]["tasks"].as_object().unwrap().is_empty()); + for field in ["input", "output"] { + let mut escaped = task.clone(); + escaped[field] = json!("../outside"); + assert!(a.call("hot_folder_configure", &json!({"config":"bad.json","settings":{"tasks":[escaped]}})).is_err()); + } + assert!(!root.join("bad.json").exists()); +} + +#[test] +fn hot_folder_tools_produce_searchable_output_and_preserve_source() { + if !pdfcraft_engine::ocr::available() { + return; + } + let root = std::env::temp_dir().join(format!("pdfcraft-hot-folder-ocr-tools-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("in")).unwrap(); + std::fs::create_dir_all(root.join("out")).unwrap(); + let mut a = Automation::new().with_root(&root).unwrap(); + let source = pdfcraft_engine::Session::new().create_from_text("source", "Hot folder searchable output").unwrap(); + std::fs::write(root.join("in/source.pdf"), source.as_slice()).unwrap(); + call( + &mut a, + "hot_folder_configure", + json!({"config":"tasks.json","settings":{"tasks":[{"id":"test","name":"Test","input":"in","output":"out","enabled":true,"interval_seconds":1,"settle_seconds":1}]}}), + ); + call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + std::thread::sleep(std::time::Duration::from_millis(1100)); + let result = call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + assert_eq!(result["tasks"]["test"]["completed"], 1, "{result}"); + assert_eq!(std::fs::read(root.join("in/source.pdf")).unwrap(), *source); + let doc = call(&mut a, "doc_open", json!({"path":"out/source.pdf"}))["doc"].clone(); + assert!(call(&mut a, "text_extract", json!({"doc":doc})).to_string().contains("searchable")); +} + +#[test] +fn hot_folder_tools_apply_page_templates_blank_archives_and_multiple_outputs() { + let root = std::env::temp_dir().join(format!("pdfcraft-hot-folder-pipeline-tools-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + for folder in ["in", "out", "blank"] { + std::fs::create_dir_all(root.join(folder)).unwrap(); + } + let session = pdfcraft_engine::Session::new(); + let text = session.create_from_text("source", "Pipeline output remains searchable").unwrap(); + let blank = session.create_blank(100.0, 100.0, 1).unwrap(); + let source = session.combine(&[("text.pdf".into(), text), ("blank.pdf".into(), blank)]).unwrap(); + std::fs::write(root.join("in/source.pdf"), source.as_slice()).unwrap(); + let mut a = Automation::new().with_root(&root).unwrap(); + let task = json!({"id":"pipeline","name":"Pipeline","input":"in","output":"out","blank_archive":"blank","remove_blank_pages":true,"enabled":true,"interval_seconds":1,"settle_seconds":1,"settings":{"recognize":false,"auto_rotate":false,"deskew":false},"preprocessing":{"pages":"1-2"},"output_plan":{"steps":[{"format":"pdf"},{"format":"txt"}]}}); + let mut escaped = task.clone(); + escaped["output_plan"]["steps"][1]["folder"] = json!("../outside"); + assert!(a.call("hot_folder_configure", &json!({"config":"bad.json","settings":{"tasks":[escaped]}})).is_err()); + call(&mut a, "hot_folder_configure", json!({"config":"tasks.json","settings":{"tasks":[task]}})); + call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + std::thread::sleep(std::time::Duration::from_millis(1100)); + let result = call(&mut a, "hot_folder_poll", json!({"config":"tasks.json"})); + assert_eq!(result["tasks"]["pipeline"]["completed"], 1, "{result}"); + assert!(root.join("out/source.pdf").exists()); + assert!(root.join("out/source.txt").exists()); + assert!(root.join("blank/source_blank_pages.pdf").exists()); + assert!(std::fs::read_to_string(root.join("out/source.txt")).unwrap().contains("Pipeline")); + assert_eq!(std::fs::read(root.join("in/source.pdf")).unwrap(), *source); +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index bcf762fc28..6c32a23872 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -13,6 +13,7 @@ pdfcraft-measure = { workspace = true } image = { version = "0.25", default-features = false, features = ["png", "jpeg", "tiff"] } pdfcraft-render = { workspace = true } pdfcraft-cos = { workspace = true } +pdfcraft-content = { workspace = true } pdfcraft-organize = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-forms = { workspace = true } @@ -41,5 +42,29 @@ idna = "1.1" icu_properties = "2.3" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = "0.11" +tempfile = "3" +ureq = { version = "3.4", default-features = false, features = ["rustls-no-provider", "_ring"] } +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +rustls-native-certs = "0.8" +zeroize = { version = "1.8", features = ["derive"] } # Already in the graph through lopdf; used only for the local UTC offset ("today"). chrono = { version = "0.4", default-features = false, features = ["clock"] } + +[target.'cfg(target_os = "windows")'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["windows-native"] } +windows-permissions = "0.2.4" + +[target.'cfg(unix)'.dependencies] +rustix = { version = "1", features = ["process"] } + +[target.'cfg(target_os = "macos")'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } + +[target.'cfg(any(target_os = "linux", target_os = "freebsd", target_os = "openbsd"))'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["async-secret-service", "async-io", "crypto-rust"] } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +rcgen = { version = "0.14", default-features = false, features = ["ring", "crypto"] } diff --git a/crates/engine/src/hot_folder.rs b/crates/engine/src/hot_folder.rs new file mode 100644 index 0000000000..e7425e5cbd --- /dev/null +++ b/crates/engine/src/hot_folder.rs @@ -0,0 +1,1576 @@ +//! Durable, opt-in local folder processing shared by the standalone app and agent tools. +//! No file watcher, network listener or worker is started by loading this module. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::hot_folder_output::{self as output, OutputPlan, OutputProof}; +use crate::hot_folder_preprocess as preprocess; +pub use crate::hot_folder_schedule::Schedule; +use crate::hot_folder_sources as sources; +use crate::{Session, ocr}; + +const MAX_INPUT: u64 = 512 * 1024 * 1024; +const MAX_ENTRIES: usize = 10_000; +const MAX_JSON: u64 = 16 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Originals { + #[default] + Keep, + Move, + Delete, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Task { + pub id: String, + pub name: String, + pub input: PathBuf, + pub output: PathBuf, + pub archive: PathBuf, + pub enabled: bool, + pub recursive: bool, + pub pdf: bool, + pub images: bool, + /// Produce JSON layout reports without invoking OCR or applying scan/image corrections. + pub analysis_only: bool, + /// Conservative: only entirely white rendered pages without text or annotations. + pub remove_blank_pages: bool, + /// Unix seconds; zero means now. Once tasks finish after their initial stable queue drains. + pub start_at: u64, + pub once: bool, + pub schedule: Schedule, + pub interval_seconds: u64, + pub settle_seconds: u64, + pub originals: Originals, + pub settings: ocr::OcrSettings, + pub output_plan: OutputPlan, + pub preprocessing: preprocess::Settings, + pub blank_archive: Option, + pub source: sources::SourceConfig, +} + +impl Default for Task { + fn default() -> Self { + Self { + id: String::new(), + name: String::new(), + input: PathBuf::new(), + output: PathBuf::new(), + archive: PathBuf::new(), + enabled: false, + recursive: false, + pdf: true, + images: true, + analysis_only: false, + remove_blank_pages: false, + start_at: 0, + once: false, + schedule: Schedule::Interval, + interval_seconds: 10, + settle_seconds: 3, + originals: Originals::Keep, + settings: ocr::OcrSettings::default(), + output_plan: OutputPlan::default(), + preprocessing: preprocess::Settings::default(), + blank_archive: None, + source: sources::SourceConfig::default(), + } + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Config { + pub tasks: Vec, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct TaskStatus { + pub completed: u64, + pub failed: u64, + pub waiting: usize, + pub next_check: u64, + pub finished: bool, + pub batch_active: bool, + /// A failed combined batch requires an explicit retry so no source is silently omitted. + pub blocked: bool, + pub last_input: Option, + pub last_output: Option, + pub error: Option, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct State { + pub updated_at: u64, + pub tasks: BTreeMap, + observations: BTreeMap, + receipts: BTreeMap, + pub log: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LogEntry { + pub at: u64, + pub task: String, + pub input: Option, + pub output: Option, + pub error: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Observation { + fingerprint: String, + since: u64, + attempted: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Receipt { + outputs: Vec, + source_hash: String, + #[serde(default)] + source_stamp: String, + #[serde(default)] + output_proofs: Vec, + /// Persisted before touching originals; a retry resumes cleanup without rerunning OCR. + #[serde(default)] + cleanup: Option, +} + +fn output_proof(path: &Path, root: Option<&Path>) -> Result { + let path = confined(path, root)?; + let metadata = fs::symlink_metadata(&path).map_err(|e| format!("saved output is unavailable; original retained: {e}"))?; + if !metadata.is_file() || metadata.len() > MAX_INPUT { + return Err("saved output is not a regular file of at most 512 MiB; original retained".into()); + } + let mut file = File::open(&path).map_err(|e| e.to_string())?.take(MAX_INPUT + 1); + let mut digest = Sha256::new(); + let mut bytes = 0_u64; + let mut buffer = [0_u8; 32 * 1024]; + loop { + let read = file.read(&mut buffer).map_err(|e| e.to_string())?; + if read == 0 { + break; + } + bytes = bytes.saturating_add(read as u64); + digest.update(buffer.get(..read).ok_or("invalid output read size")?); + } + if bytes != metadata.len() { + return Err("saved output changed during verification; original retained".into()); + } + Ok(OutputProof { path, bytes, sha256: digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect() }) +} + +fn verify_outputs(receipt: &Receipt, root: Option<&Path>) -> Result<(), String> { + if receipt.output_proofs.is_empty() || receipt.output_proofs.len() != receipt.outputs.len() { + return Err("saved-output receipt has no integrity proofs; restore verified results before source cleanup".into()); + } + for (path, proof) in receipt.outputs.iter().zip(&receipt.output_proofs) { + if confined(path, root)? != proof.path || output_proof(path, root)? != *proof { + return Err("saved output was replaced or changed; original retained".into()); + } + } + Ok(()) +} + +/// On Unix a synced temporary file does not make its renamed directory entry durable. +pub(crate) fn sync_parent(path: &Path) -> Result<(), String> { + #[cfg(unix)] + File::open(path.parent().ok_or("durable file needs a parent folder")?) + .and_then(|directory| directory.sync_all()) + .map_err(|e| format!("cannot persist folder entry; original retained: {e}"))?; + #[cfg(not(unix))] + let _ = path; + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum PendingLocalAction { + Delete, + Move { archive: PathBuf, relative: PathBuf }, +} + +pub fn now() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |v| v.as_secs()) +} + +pub fn load_config(path: &Path) -> Result { + read_json(path) +} + +/// Create a stopped configuration without replacing an existing file. Installers call this +/// as the chosen service account, never as root in another user's writable directory. +pub fn initialize_config(path: &Path) -> Result<(), String> { + fn directory(path: &Path) -> Result<(), String> { + match fs::symlink_metadata(path) { + Ok(meta) if meta.is_dir() && !meta.file_type().is_symlink() => return Ok(()), + Ok(_) => return Err("configuration parent must be a real directory".into()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.to_string()), + } + directory(path.parent().ok_or("configuration needs an absolute parent")?)?; + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + #[cfg(not(unix))] + let _ = &mut builder; + builder.create(path).map_err(|e| e.to_string())?; + sync_parent(path) + } + if !path.is_absolute() { + return Err("configuration path must be absolute".into()); + } + directory(path.parent().ok_or("configuration needs a parent")?)?; + let path = confined(path, None)?; + if path.exists() { + return validate(&load_config(&path)?, None); + } + let mut temp = tempfile::NamedTempFile::new_in(path.parent().ok_or("configuration needs a parent")?).map_err(|e| e.to_string())?; + temp.write_all(b"{\"tasks\":[]}\n").and_then(|_| temp.as_file().sync_all()).map_err(|e| e.to_string())?; + match temp.persist_noclobber(&path) { + Ok(_) => sync_parent(&path), + Err(e) if e.error.kind() == std::io::ErrorKind::AlreadyExists => validate(&load_config(&path)?, None), + Err(e) => Err(e.to_string()), + } +} + +pub fn save_config(path: &Path, config: &Config, root: Option<&Path>) -> Result<(), String> { + validate(config, root)?; + confined(path, root)?; + save_json(path, config) +} + +pub fn state_path(config: &Path) -> PathBuf { + config.with_extension("state.json") +} + +pub fn read_state(config: &Path) -> Result { + let path = state_path(config); + if !path.exists() { + return Ok(State::default()); + } + read_json(&path) +} + +pub fn record_worker_error(config: &Path, error: Option<&str>) -> Result<(), String> { + let path = config.with_extension("error.json"); + if let Some(error) = error { + save_json(&path, &error) + } else { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.to_string()), + } + } +} + +pub fn read_worker_error(config: &Path) -> Result, String> { + let path = config.with_extension("error.json"); + if !path.exists() { + return Ok(None); + } + read_json(&path).map(Some) +} + +/// Canonicalize before opening anything; reject symbolic links on the final path and root escapes. +pub fn confined(path: &Path, root: Option<&Path>) -> Result { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + root.map(Path::to_path_buf).unwrap_or(std::env::current_dir().map_err(|e| e.to_string())?).join(path) + }; + if let Some(root) = root { + let root = root.canonicalize().map_err(|e| e.to_string())?; + // Reject '..' before filesystem access; a confined call cannot probe outside the root. + if absolute.components().any(|c| matches!(c, std::path::Component::ParentDir)) || !lexically_inside(&absolute, &root) { + return Err("hot folder path is outside the allowed directory".into()); + } + } + if fs::symlink_metadata(&absolute).is_ok_and(|m| m.file_type().is_symlink()) { + return Err(format!("symbolic links are not allowed: {}", path.display())); + } + let result = if absolute.exists() { + absolute.canonicalize() + } else { + absolute.parent().ok_or("path needs a parent folder")?.canonicalize().map(|p| p.join(absolute.file_name().unwrap_or_default())) + } + .map_err(|e| format!("{}: {e}", path.display()))?; + if let Some(root) = root + && !result.starts_with(root.canonicalize().map_err(|e| e.to_string())?) + { + return Err("hot folder path is outside the allowed directory".into()); + } + Ok(result) +} + +fn lexically_inside(path: &Path, root: &Path) -> bool { + #[cfg(windows)] + { + let normalize = |p: &Path| p.to_string_lossy().trim_start_matches(r"\\?\").replace('/', "\\").to_ascii_lowercase(); + let path = normalize(path); + let root = normalize(root); + path == root || path.starts_with(&format!("{}\\", root.trim_end_matches('\\'))) + } + #[cfg(not(windows))] + path.starts_with(root) +} + +/// All sources and destinations must already exist and be disjoint, including across tasks. +/// This prevents recursive output loops and destructive source policies affecting another task. +pub fn validate(config: &Config, root: Option<&Path>) -> Result<(), String> { + if config.tasks.len() > 32 { + return Err("at most 32 hot folder tasks are allowed".into()); + } + let mut ids = BTreeSet::new(); + let mut sources = Vec::new(); + let mut destinations = Vec::new(); + for t in &config.tasks { + if t.id.is_empty() || t.id.len() > 100 || !t.id.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') || !ids.insert(&t.id) { + return Err("task IDs must be unique, using 1..100 ASCII letters, digits, '-' or '_'".into()); + } + if t.name.trim().is_empty() || t.name.len() > 200 { + return Err("task name is required (at most 200 bytes)".into()); + } + if t.input.as_os_str().is_empty() || t.output.as_os_str().is_empty() || t.originals == Originals::Move && t.archive.as_os_str().is_empty() { + return Err("choose an input, output and (when moving originals) archive folder".into()); + } + if root.is_none() && (!t.input.is_absolute() || !t.output.is_absolute() || t.originals == Originals::Move && !t.archive.is_absolute()) { + return Err("folder paths must be absolute; use the folder picker".into()); + } + if !t.pdf && !t.images { + return Err("choose PDF, images, or both".into()); + } + if !(1..=86400).contains(&t.interval_seconds) || !(1..=300).contains(&t.settle_seconds) { + return Err("check interval must be 1..86400 seconds; settling time 1..300 seconds".into()); + } + let s = &t.settings; + t.schedule.validate()?; + t.preprocessing.validate()?; + if !s.dpi.is_finite() + || !(72.0..=600.0).contains(&s.dpi) + || !s.crop_margin.is_finite() + || !(0.0..=20.0).contains(&s.crop_margin) + || !ocr::LANGUAGES.iter().any(|l| l.0 == s.language) + { + return Err("invalid OCR language, resolution or crop margin".into()); + } + let input = confined(&t.input, root)?; + if let sources::SourceConfig::Outlook(source) = &t.source + && !matches!(source.action, sources::SourceAction::Keep) + && (!t.pdf || !t.images) + { + return Err( + "Outlook move/delete requires PDF and images: every supported attachment must finish before changing the whole message".into() + ); + } + sources::validate(&t.source, &input, root)?; + let output = confined(&t.output, root)?; + t.output_plan.validate(&output, root)?; + if !input.is_dir() || !output.is_dir() { + return Err("input and output must be existing folders".into()); + } + if output.starts_with(&input) || input.starts_with(&output) { + return Err("input, output and archive folders must not contain one another".into()); + } + if t.enabled { + sources.push(input.clone()); + sources.extend(sources::local_paths(&t.source, root)?); + if let sources::SourceConfig::Project(source) = &t.source + && let sources::SourceAction::Move { destination } = &source.action + { + destinations.push(confined(Path::new(destination), root)?); + } + destinations.push(output); + } + if let Some(folder) = &t.blank_archive { + if root.is_none() && !folder.is_absolute() { + return Err("blank page archive must be absolute".into()); + } + let folder = confined(folder, root)?; + if !folder.is_dir() || folder.starts_with(&input) || input.starts_with(&folder) { + return Err("blank page archive must be an existing folder disjoint from the input".into()); + } + if t.enabled { + destinations.push(folder); + } + } + for step in &t.output_plan.steps { + if let Some(folder) = &step.folder { + if root.is_none() && !folder.is_absolute() { + return Err("output step folders must be absolute".into()); + } + let folder = confined(folder, root)?; + if folder.starts_with(&input) || input.starts_with(&folder) { + return Err("output step folders must not overlap an input folder".into()); + } + if t.enabled { + destinations.push(folder); + } + } + } + if t.originals == Originals::Move { + let archive = confined(&t.archive, root)?; + if !archive.is_dir() { + return Err("archive must be an existing folder".into()); + } + if archive.starts_with(&input) || input.starts_with(&archive) { + return Err("input, output and archive folders must not contain one another".into()); + } + if t.enabled { + destinations.push(archive); + } + } + } + for source in &sources { + if destinations.iter().any(|d| d.starts_with(source) || source.starts_with(d)) { + return Err("input, output and archive folders must not contain one another".into()); + } + } + for (i, source) in sources.iter().enumerate() { + if sources.iter().skip(i + 1).any(|d| d.starts_with(source) || source.starts_with(d)) { + return Err("task input folders must not overlap".into()); + } + } + Ok(()) +} + +fn read_json Deserialize<'a>>(path: &Path) -> Result { + let file = File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + if file.metadata().map_err(|e| e.to_string())?.len() > MAX_JSON { + return Err("hot folder JSON exceeds 16 MiB".into()); + } + serde_json::from_reader(file.take(MAX_JSON)).map_err(|e| format!("{}: {e}", path.display())) +} + +fn save_json(path: &Path, value: &T) -> Result<(), String> { + let previous = match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + return Err("configuration and journal targets must be regular files".into()); + } + Ok(metadata) => Some(metadata), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(e.to_string()), + }; + let bytes = serde_json::to_vec_pretty(value).map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_JSON { + return Err("hot folder journal is full; archive the completed task and its configuration".into()); + } + let parent = path.parent().ok_or("missing parent folder")?; + let mut temp = tempfile::NamedTempFile::new_in(parent).map_err(|e| e.to_string())?; + #[cfg(unix)] + if let Some(metadata) = &previous { + use std::os::unix::fs::MetadataExt; + // Keep a service-owned configuration service-readable even when an administrator GUI + // performs the atomic replacement. Failure leaves the previous file untouched. + std::os::unix::fs::chown(temp.path(), Some(metadata.uid()), Some(metadata.gid())).map_err(|e| e.to_string())?; + temp.as_file().set_permissions(metadata.permissions()).map_err(|e| e.to_string())?; + } + #[cfg(not(unix))] + let _ = previous; + temp.write_all(&bytes).and_then(|_| temp.as_file().sync_all()).map_err(|e| e.to_string())?; + temp.persist(path).map_err(|e| e.to_string())?; + sync_parent(path) +} + +/// Run one bounded scheduler pass; persists observations, receipts, errors and schedule state. +/// One file or one bounded combined batch per task is processed per pass. +/// The lock is an OS file lock, released even if the process exits unexpectedly. +pub fn poll(config_path: &Path, root: Option<&Path>) -> Result { + poll_cancelable(config_path, root, &|| false) +} + +/// Service-aware form: cancellation between files and OCR pages discards partial outputs. +pub fn poll_cancelable(config_path: &Path, root: Option<&Path>, should_stop: &dyn Fn() -> bool) -> Result { + let path = confined(config_path, root)?; + let state_file = confined(&state_path(&path), root)?; + let lock_path = confined(&path.with_extension("lock"), root)?; + let lock = OpenOptions::new().create(true).truncate(false).read(true).write(true).open(lock_path).map_err(|e| e.to_string())?; + lock.try_lock().map_err(|_| "another Hot Folder worker is processing this configuration".to_string())?; + let config = load_config(&path)?; + validate(&config, root)?; + let mut state = read_state(&path)?; + let current = now(); + for task in &config.tasks { + if should_stop() { + break; + } + if !task.enabled || current < task.start_at { + continue; + } + let status = state.tasks.entry(task.id.clone()).or_default(); + if !task.once && task.schedule != Schedule::Interval && status.next_check == 0 && !status.batch_active { + status.next_check = task.schedule.next_after(current.max(task.start_at).saturating_sub(1))?; + } + if status.finished || status.blocked || current < status.next_check { + continue; + } + status.next_check = current.saturating_add(task.interval_seconds); + status.batch_active = true; + if let Err(error) = step(task, &mut state, current, &path, root, should_stop) { + let status = state.tasks.entry(task.id.clone()).or_default(); + status.error = Some(error.clone()); + let snapshot = serde_json::to_vec(task).map_err(|e| e.to_string())?; + let still_enabled = task_unchanged(&path, &task.id, &snapshot); + status.blocked = task.output_plan.needs_batch() && !should_stop() && still_enabled; + status.failed = status.failed.saturating_add(1); + state.log.push(LogEntry { at: now(), task: task.id.clone(), input: status.last_input.clone(), output: None, error: Some(error) }); + } + if state.log.len() > 1000 { + state.log.drain(..state.log.len().saturating_sub(1000)); + } + let status = state.tasks.entry(task.id.clone()).or_default(); + if status.waiting == 0 { + status.batch_active = false; + if !task.once && task.schedule != Schedule::Interval { + status.next_check = task.schedule.next_after(now())?; + } + } + state.updated_at = now(); + save_json(&state_file, &state)?; + } + state.updated_at = current; + save_json(&state_file, &state)?; + Ok(state) +} + +fn task_unchanged(config: &Path, id: &str, snapshot: &[u8]) -> bool { + load_config(config).is_ok_and(|config| { + config.tasks.iter().any(|task| task.id == id && task.enabled && serde_json::to_vec(task).is_ok_and(|current| current == snapshot)) + }) +} + +fn fingerprint(path: &Path) -> Result { + let meta = fs::symlink_metadata(path).map_err(|e| e.to_string())?; + if !meta.is_file() || meta.file_type().is_symlink() { + return Err("input is not a regular file".into()); + } + let modified = meta.modified().map_err(|e| e.to_string())?.duration_since(UNIX_EPOCH).map_err(|e| e.to_string())?.as_nanos(); + Ok(format!("{}:{modified}", meta.len())) +} + +fn files(task: &Task, root: Option<&Path>) -> Result, String> { + let input = confined(&task.input, root)?; + let mut pending = vec![(input.clone(), 0usize)]; + let mut out = Vec::new(); + let mut count = 0usize; + while let Some((dir, depth)) = pending.pop() { + for entry in fs::read_dir(dir).map_err(|e| e.to_string())? { + count = count.saturating_add(1); + if count > MAX_ENTRIES { + return Err("input folder exceeds 10,000 entries".into()); + } + let entry = entry.map_err(|e| e.to_string())?; + let kind = entry.file_type().map_err(|e| e.to_string())?; + if kind.is_symlink() { + continue; + } + let path = entry.path(); + if !confined(&path, root)?.starts_with(&input) { + return Err("input escaped its folder".into()); + } + if kind.is_dir() && task.recursive { + if depth >= 32 { + return Err("input folder nesting exceeds 32 levels".into()); + } + pending.push((path, depth + 1)); + } else if kind.is_file() { + let ext = path.extension().unwrap_or_default().to_string_lossy().to_ascii_lowercase(); + if task.pdf && ext == "pdf" || task.images && matches!(ext.as_str(), "png" | "jpg" | "jpeg" | "tif" | "tiff" | "bmp") { + out.push(path); + } + } + } + } + out.sort(); + Ok(out) +} + +struct Prepared { + source: PathBuf, + key: String, + stamp: String, + hash: String, + relative: PathBuf, + processed: Option>>, + blank_pages: Option>>, + remote: Option, +} + +fn remove_staged(path: &Path, expected: &str) -> Result<(), String> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.to_string()), + }; + if !metadata.is_file() || metadata.len() > MAX_INPUT { + return Err("staging cleanup refused a changed or nonregular file".into()); + } + let mut file = File::open(path).map_err(|e| e.to_string())?.take(MAX_INPUT + 1); + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 32 * 1024]; + loop { + let read = file.read(&mut buffer).map_err(|e| e.to_string())?; + if read == 0 { + break; + } + digest.update(buffer.get(..read).ok_or("invalid staging read size")?); + } + let hash: String = digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect(); + if hash != expected { + return Err("staged file changed after processing; retained for inspection".into()); + } + fs::remove_file(path).map_err(|e| format!("outputs and source acknowledgement saved, but staging cleanup failed: {e}")) +} + +fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: Option<&Path>, should_stop: &dyn Fn() -> bool) -> Result<(), String> { + if state.observations.len() >= MAX_ENTRIES || state.receipts.len() >= MAX_ENTRIES { + return Err("task journal reached 10,000 entries; archive it before continuing".into()); + } + let input = confined(&task.input, root)?; + let output = confined(&task.output, root)?; + let snapshot = serde_json::to_vec(task).map_err(|e| e.to_string())?; + let task_stopped = || should_stop() || !task_unchanged(config_path, &task.id, &snapshot); + let store = match &task.source { + sources::SourceConfig::Ftp(_) | sources::SourceConfig::Outlook(_) => sources::CredentialStore::for_config(config_path, root)?, + _ => sources::CredentialStore::OsVault, + }; + let mut remote = BTreeMap::new(); + let mut acknowledgement_proofs = Vec::new(); + let candidates = if matches!(task.source, sources::SourceConfig::Local) { + files(task, root)? + } else { + let staged = sources::stage(&task.source, &input, root, &task_stopped, &store)?; + if staged.has_more && task.output_plan.needs_batch() { + return Err("combined remote queue exceeds 128 files or 512 MiB; choose a smaller source or per-file outputs".into()); + } + for warning in staged.warnings { + state.log.push(LogEntry { at: now(), task: task.id.clone(), input: None, output: None, error: Some(warning) }); + } + for item in &staged.pending_acknowledgements { + let path = confined(&input.join(&item.staged_relative), root)?; + let key = format!("{}:{}:{}", task.id, path.display(), item.sha256); + let receipt = state.receipts.get(&key).ok_or("pending source acknowledgement has no saved-output receipt; original retained")?; + verify_outputs(receipt, root)?; + } + let mut items = staged.items; + items.extend(staged.pending_acknowledgements); + // A final Outlook acknowledgement may consume the whole message. Verify every + // earlier ready attachment's outputs before acknowledging any of its peers. + for item in &items { + let path = confined(&input.join(&item.staged_relative), root)?; + let key = format!("{}:{}:{}", task.id, path.display(), item.sha256); + if let Some(receipt) = state.receipts.get(&key) { + verify_outputs(receipt, root)?; + acknowledgement_proofs.push(receipt.clone()); + } + } + if !matches!(task.source, sources::SourceConfig::Project(_)) { + items.sort_by(|a, b| a.relative.cmp(&b.relative)); + } + let mut paths = Vec::new(); + for item in items { + let path = confined(&input.join(&item.staged_relative), root)?; + if !path.starts_with(&input) { + return Err("staged source escaped its inbox".into()); + } + let key = format!("{}:{}", task.id, path.display()); + // Recover a failed/interrupted acknowledgement without rerunning OCR or saving duplicates. + if state.receipts.contains_key(&format!("{key}:{}", item.sha256)) { + for receipt in &acknowledgement_proofs { + verify_outputs(receipt, root)?; + } + sources::acknowledge(&task.source, &input, &item, &item.sha256, root, &task_stopped, &store)?; + remove_staged(&path, &item.sha256)?; + continue; + } + let ext = item.relative.extension().unwrap_or_default().to_string_lossy().to_ascii_lowercase(); + if task.pdf && ext == "pdf" || task.images && matches!(ext.as_str(), "png" | "jpg" | "jpeg" | "tif" | "tiff" | "bmp") { + remote.insert(path.clone(), item); + paths.push(path); + } + } + paths + }; + let mut ready = Vec::new(); + let mut waiting = 0usize; + let batch = task.output_plan.needs_batch(); + for path in candidates { + let key = format!("{}:{}", task.id, path.display()); + let stamp = fingerprint(&path).map_err(|e| format!("{}: {e}", path.display()))?; + let observation = + state.observations.entry(key.clone()).or_insert_with(|| Observation { fingerprint: stamp.clone(), since: current, attempted: false }); + if observation.fingerprint != stamp { + *observation = Observation { fingerprint: stamp.clone(), since: current, attempted: false }; + } + let pending_cleanup = state + .receipts + .iter() + .any(|(receipt_key, receipt)| receipt_key.starts_with(&format!("{key}:")) && receipt.source_stamp == stamp && receipt.cleanup.is_some()); + if observation.attempted && !pending_cleanup { + continue; + } + waiting = waiting.saturating_add(1); + if (batch || ready.is_empty()) && current.saturating_sub(observation.since) >= task.settle_seconds { + ready.push((path, key, stamp)); + } + } + let status = state.tasks.entry(task.id.clone()).or_default(); + status.waiting = waiting; + if ready.is_empty() && waiting == 0 && task.once { + status.finished = true; + } + // A grouped result includes the complete currently visible queue; never consume an + // unfinished copy or publish a misleading partial group while another input is settling. + if ready.is_empty() || batch && ready.len() != waiting { + return Ok(()); + } + if ready.len() > 128 { + return Err("combined batch exceeds 128 files; split the task or use one output per file".into()); + } + let mut prepared = Vec::new(); + let mut total = 0u64; + for (source, key, stamp) in ready { + if task_stopped() { + return Err("task stopped; no partial output was saved".into()); + } + state.tasks.entry(task.id.clone()).or_default().last_input = Some(source.clone()); + let file = File::open(&source).map_err(|e| e.to_string())?; + let length = file.metadata().map_err(|e| e.to_string())?.len(); + total = total.saturating_add(length); + if length == 0 || total > MAX_INPUT { + if let Some(o) = state.observations.get_mut(&key) { + o.attempted = true; + } + return Err("input or combined batch must contain 1 byte to 512 MiB; sources retained".into()); + } + let mut bytes = Vec::new(); + file.take(MAX_INPUT + 1).read_to_end(&mut bytes).map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_INPUT || fingerprint(&source)? != stamp { + return Err("input changed while being read; waiting for a stable copy".into()); + } + let hash: String = Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect(); + let receipt_key = format!("{key}:{hash}"); + if let Some(receipt) = state.receipts.get(&receipt_key).cloned() { + if let Some(cleanup) = &receipt.cleanup { + verify_outputs(&receipt, root)?; + finish_original(&source, &hash, cleanup, task, root, &task_stopped)?; + if let Some(receipt) = state.receipts.get_mut(&receipt_key) { + receipt.cleanup = None; + } + save_json(&state_path(config_path), state)?; + } + if let Some(o) = state.observations.get_mut(&key) { + o.attempted = true; + } + state.tasks.entry(task.id.clone()).or_default().waiting = state.tasks.get(&task.id).map_or(0, |s| s.waiting.saturating_sub(1)); + continue; + } + let processed = match prepare(task, &source, bytes.clone(), config_path, &task_stopped) { + Ok(bytes) => bytes, + Err(e) => { + if !task_stopped() + && let Some(o) = state.observations.get_mut(&key) + { + o.attempted = true; + } + return Err(e); + } + }; + if fingerprint(&source)? != stamp { + return Err("input changed during processing; sources retained".into()); + } + prepared.push(Prepared { + relative: remote.get(&source).map(|item| item.relative.clone()).unwrap_or(source.strip_prefix(&input).map_err(|e| e.to_string())?.into()), + remote: remote.get(&source).cloned(), + processed: processed.document, + blank_pages: processed.blank_pages, + source, + key, + stamp, + hash, + }); + } + if prepared.is_empty() { + return Ok(()); + } + let stopped = &task_stopped; + if stopped() { + return Err("task stopped; no partial output was saved".into()); + } + for file in &prepared { + if fingerprint(&file.source)? != file.stamp { + return Err("input changed before saving; sources retained".into()); + } + } + let inputs: Vec<_> = prepared + .iter() + .filter_map(|p| p.processed.as_ref().map(|bytes| output::Input { relative: p.relative.clone(), bytes: bytes.clone() })) + .collect(); + let primary = if inputs.is_empty() { + Ok(output::OutputReceipt::default()) + } else { + let mut plan = task.output_plan.clone(); + for step in &mut plan.steps { + if task.analysis_only { + step.format = output::Format::Analysis; + } + if step.format == output::Format::Epub && step.language == "und" { + step.language = task.settings.language.clone(); + } + } + output::save(&inputs, &output, &plan, &task.name, current, root, &stopped) + }; + let mut result = match primary { + Ok(result) => result, + Err(error) => { + for path in error.outputs { + state.log.push(LogEntry { at: now(), task: task.id.clone(), input: None, output: Some(path), error: Some(error.message.clone()) }); + } + for file in &prepared { + if let Some(o) = state.observations.get_mut(&file.key) { + o.attempted = !stopped(); + } + } + return Err(format!("{}; all originals retained", error.message)); + } + }; + if let Some(archive) = &task.blank_archive { + let blanks: Vec<_> = prepared + .iter() + .filter_map(|p| p.blank_pages.as_ref().map(|bytes| output::Input { relative: p.relative.clone(), bytes: bytes.clone() })) + .collect(); + if !blanks.is_empty() { + let plan = OutputPlan { steps: vec![output::SaveStep { name_template: "{name}_blank_pages".into(), ..Default::default() }] }; + match output::save(&blanks, archive, &plan, &task.name, current, root, &stopped) { + Ok(saved) => { + result.outputs.extend(saved.outputs); + result.proofs.extend(saved.proofs); + result.warnings.extend(saved.warnings); + } + Err(error) => { + for path in result.outputs.into_iter().chain(error.outputs) { + state.log.push(LogEntry { + at: now(), + task: task.id.clone(), + input: None, + output: Some(path), + error: Some(error.message.clone()), + }); + } + for file in &prepared { + if let Some(o) = state.observations.get_mut(&file.key) { + o.attempted = !stopped(); + } + } + return Err(format!("blank-page archive failed: {}; all originals retained", error.message)); + } + } + } + } + for proof in &result.proofs { + if output_proof(&proof.path, root)? != *proof { + return Err("published output no longer matches the export; all originals retained".into()); + } + } + for file in &prepared { + let cleanup = if file.remote.is_some() { + None + } else { + match task.originals { + Originals::Keep => None, + Originals::Delete => Some(PendingLocalAction::Delete), + Originals::Move => Some(PendingLocalAction::Move { archive: confined(&task.archive, root)?, relative: file.relative.clone() }), + } + }; + state.receipts.insert( + format!("{}:{}", file.key, file.hash), + Receipt { + outputs: result.outputs.clone(), + source_hash: file.hash.clone(), + source_stamp: file.stamp.clone(), + output_proofs: result.proofs.clone(), + cleanup, + }, + ); + if let Some(o) = state.observations.get_mut(&file.key) { + o.attempted = true; + } + state.log.push(LogEntry { + at: now(), + task: task.id.clone(), + input: Some(file.source.clone()), + output: result.outputs.first().cloned(), + error: None, + }); + } + for path in result.outputs.iter().skip(1) { + state.log.push(LogEntry { at: now(), task: task.id.clone(), input: None, output: Some(path.clone()), error: None }); + } + for warning in result.warnings { + state.log.push(LogEntry { at: now(), task: task.id.clone(), input: None, output: None, error: Some(warning) }); + } + let status = state.tasks.entry(task.id.clone()).or_default(); + status.last_output = result.outputs.first().cloned(); + status.completed = status.completed.saturating_add(prepared.len() as u64); + status.error = None; + status.waiting = status.waiting.saturating_sub(prepared.len()); + // All outputs and their receipts are durable before any source move/delete is attempted. + save_json(&state_path(config_path), state)?; + for file in prepared { + if stopped() { + return Err("task stopped after outputs were saved; remaining original actions will resume on retry".into()); + } + let receipt_key = format!("{}:{}", file.key, file.hash); + if file.remote.is_some() || state.receipts.get(&receipt_key).is_some_and(|r| r.cleanup.is_some()) { + verify_outputs(state.receipts.get(&receipt_key).ok_or("output receipt unavailable; original retained")?, root)?; + } + if let Some(remote) = &file.remote { + // A previous ready attachment's output may have changed during lengthy OCR + // of this final attachment. Recheck the whole pending group at the action. + for receipt in &acknowledgement_proofs { + verify_outputs(receipt, root)?; + } + sources::acknowledge(&task.source, &input, remote, &file.hash, root, &stopped, &store)?; + remove_staged(&file.source, &file.hash)?; + continue; + } + let key = format!("{}:{}", file.key, file.hash); + if let Some(action) = state.receipts.get(&key).and_then(|r| r.cleanup.clone()) { + finish_original(&file.source, &file.hash, &action, task, root, &stopped)?; + if let Some(receipt) = state.receipts.get_mut(&key) { + receipt.cleanup = None; + } + save_json(&state_path(config_path), state)?; + } + } + Ok(()) +} + +fn prepare( + task: &Task, + source: &Path, + bytes: Vec, + config_path: &Path, + should_stop: &dyn Fn() -> bool, +) -> Result { + let name = source.file_name().unwrap_or_default().to_string_lossy(); + let pdf = if source.extension().is_some_and(|e| e.eq_ignore_ascii_case("pdf")) { + Arc::new(bytes) + } else { + Session::new().create_from_images(&[(name.to_string(), bytes)]).map_err(|e| e.to_string())? + }; + if task.analysis_only { + let selected = if task.preprocessing.pages.trim().is_empty() { + pdf + } else { + // COS-only page selection; do not open an interactive document or run its scripts. + Session::new().combine_ranges(&[(name.to_string(), pdf, Some(task.preprocessing.pages.clone()))]).map_err(|e| e.to_string())? + }; + return Ok(preprocess::BlankResult { document: Some(selected), blank_pages: None }); + } + let pdf = preprocess::prepare(pdf, &task.preprocessing, f64::from(task.settings.dpi), should_stop)?; + let split = if task.remove_blank_pages { + preprocess::separate_blank_pages(pdf, should_stop)? + } else { + preprocess::BlankResult { document: Some(pdf), blank_pages: None } + }; + let blanks = split.blank_pages; + let Some(pdf) = split.document else { + return if task.blank_archive.is_some() { + Ok(preprocess::BlankResult { document: None, blank_pages: blanks }) + } else { + Err("all pages are blank; select a blank-page archive to keep them separately".into()) + }; + }; + if !task.settings.recognize && !task.settings.auto_rotate && !task.settings.deskew && !task.settings.auto_crop { + return Ok(preprocess::BlankResult { document: Some(pdf), blank_pages: blanks }); + } + let model = ocr::engine()?; + let mut cancelled = false; + let mut session = Session::new(); + let id = session.open(name.as_ref(), None, pdf, None).map_err(|e| e.to_string())?; + let doc = session.get(id).ok_or("document unavailable")?; + if let Some(reason) = &doc.read_only_reason { + return Err(reason.clone()); + } + let infos = doc.info.pages.clone(); + let job = session.ocr_job(id, &[], task.settings.clone()).ok_or("document unavailable")?; + let mut pages = job.run(&model, |_, _| { + let enabled = !should_stop() && load_config(config_path).is_ok_and(|c| c.tasks.iter().any(|t| t.id == task.id && t.enabled)); + cancelled |= !enabled; + enabled + }); + preprocess::constrain_words(&mut pages, &infos, &task.preprocessing.regions); + + if cancelled || should_stop() { + return Err("task stopped; no partial output was saved".into()); + } + if let Some(why) = pages.iter().filter_map(|p| p.skipped.as_deref()).find(|s| *s != "the page already contains text") { + return Err(format!("scan processing failed; sources retained: {why}")); + } + session.apply_ocr(id, &pages).map_err(|e| e.to_string())?; + Ok(preprocess::BlankResult { document: Some(session.save_bytes(id).map_err(|e| e.to_string())?), blank_pages: blanks }) +} + +#[cfg(test)] +fn remove_blank_pages(bytes: Arc>) -> Result>, String> { + preprocess::separate_blank_pages(bytes, &|| false)?.document.ok_or_else(|| "all pages are blank".into()) +} + +fn safe_subfolder(base: &Path, relative: &Path) -> Result<(), String> { + let mut path = base.to_path_buf(); + for part in relative.components() { + if !matches!(part, std::path::Component::Normal(_)) { + return Err("invalid output subfolder".into()); + } + path.push(part); + if fs::symlink_metadata(&path).is_ok_and(|m| m.file_type().is_symlink()) { + return Err("output subfolder is a symbolic link".into()); + } + fs::create_dir_all(&path).map_err(|e| e.to_string())?; + sync_parent(&path)?; + if !path.canonicalize().map_err(|e| e.to_string())?.starts_with(base) { + return Err("output subfolder escaped its folder".into()); + } + } + Ok(()) +} + +fn finish_original( + source: &Path, + hash: &str, + action: &PendingLocalAction, + task: &Task, + root: Option<&Path>, + stopped: &dyn Fn() -> bool, +) -> Result<(), String> { + if stopped() { + return Err("task stopped; original retained".into()); + } + // A changed task policy must never retroactively authorize deletion of old inputs. + match action { + PendingLocalAction::Delete if task.originals != Originals::Delete => return Ok(()), + PendingLocalAction::Move { archive, .. } if task.originals != Originals::Move || confined(&task.archive, root)? != *archive => return Ok(()), + _ => {} + } + if !fs::symlink_metadata(source).map_err(|e| e.to_string())?.is_file() { + return Err("original is not a regular file".into()); + } + let mut bytes = Vec::new(); + File::open(source).map_err(|e| e.to_string())?.take(MAX_INPUT + 1).read_to_end(&mut bytes).map_err(|e| e.to_string())?; + let actual: String = Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect(); + if bytes.len() as u64 > MAX_INPUT || actual != hash { + return Err("source changed after output was saved; original retained".into()); + } + if let PendingLocalAction::Move { archive, relative } = action { + let archive = confined(archive, root)?; + let parent = relative.parent().unwrap_or(Path::new("")); + safe_subfolder(&archive, parent)?; + save_unique( + &archive.join(parent), + source.file_stem().unwrap_or_default().to_string_lossy().as_ref(), + source.extension().unwrap_or_default().to_string_lossy().as_ref(), + &bytes, + )?; + } + if stopped() { + return Err("task stopped after archive copy; original retained".into()); + } + remove_staged(source, hash) +} + +fn save_unique(folder: &Path, stem: &str, extension: &str, bytes: &[u8]) -> Result { + let stem: String = stem.chars().take(100).collect(); + let mut temp = tempfile::NamedTempFile::new_in(folder).map_err(|e| e.to_string())?; + temp.write_all(bytes).and_then(|_| temp.as_file().sync_all()).map_err(|e| e.to_string())?; + for n in 0..1000 { + let name = if n == 0 { format!("{stem}.{extension}") } else { format!("{stem} ({n}).{extension}") }; + let target = folder.join(name); + if let Ok(meta) = fs::symlink_metadata(&target) { + if meta.is_file() && meta.len() == bytes.len() as u64 { + let mut existing = Vec::new(); + File::open(&target).map_err(|e| e.to_string())?.take(MAX_INPUT + 1).read_to_end(&mut existing).map_err(|e| e.to_string())?; + if existing == bytes { + sync_parent(&target)?; + return Ok(target); + } + } + continue; + } + match temp.persist_noclobber(&target) { + Ok(_) => { + sync_parent(&target)?; + return Ok(target); + } + Err(e) if e.error.kind() == std::io::ErrorKind::AlreadyExists => temp = e.file, + Err(e) => return Err(e.to_string()), + } + } + Err("too many existing output filenames; no file was overwritten".into()) +} + +/// Retry failed inputs / reschedule completed once tasks without forgetting successful receipts. +pub fn reset(config_path: &Path, task_id: &str, root: Option<&Path>) -> Result<(), String> { + reset_for_run(config_path, task_id, root, false) +} + +pub fn run_now(config_path: &Path, task_id: &str, root: Option<&Path>) -> Result<(), String> { + reset_for_run(config_path, task_id, root, true) +} + +fn reset_for_run(config_path: &Path, task_id: &str, root: Option<&Path>, immediate: bool) -> Result<(), String> { + let path = confined(config_path, root)?; + if !load_config(&path)?.tasks.iter().any(|t| t.id == task_id) { + return Err("no such Hot Folder task".into()); + } + let lock_path = confined(&path.with_extension("lock"), root)?; + let lock = OpenOptions::new().create(true).truncate(false).read(true).write(true).open(lock_path).map_err(|e| e.to_string())?; + lock.try_lock().map_err(|_| "worker is busy; stop the task and retry shortly".to_string())?; + let mut state = read_state(&path)?; + state.tasks.remove(task_id); + if immediate { + state.tasks.insert(task_id.into(), TaskStatus { batch_active: true, ..Default::default() }); + } + state.observations.retain(|k, _| !k.starts_with(&format!("{task_id}:"))); + save_json(&confined(&state_path(&path), root)?, &state) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn setup() -> (tempfile::TempDir, PathBuf, Task) { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("input"); + let output = dir.path().join("output"); + let archive = dir.path().join("archive"); + for path in [&input, &output, &archive] { + fs::create_dir(path).unwrap(); + } + let task = Task { id: "scan".into(), name: "Scan task".into(), input, output, archive, enabled: true, ..Default::default() }; + let path = dir.path().join("tasks.json"); + save_config(&path, &Config { tasks: vec![task.clone()] }, None).unwrap(); + (dir, path, task) + } + + fn text_pdf() -> Arc> { + Session::new().create_from_text("test", "A searchable original remains intact").unwrap() + } + + fn settled(task: &Task, state: &mut State, path: &Path) -> Result<(), String> { + step(task, state, 10, path, None, &|| false)?; + step(task, state, 13, path, None, &|| false) + } + + #[test] + fn hot_folder_initialize_is_atomic_and_preserves_existing_configs() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nested/tasks.json"); + initialize_config(&path).unwrap(); + assert!(load_config(&path).unwrap().tasks.is_empty()); + fs::write(&path, b"broken existing settings").unwrap(); + assert!(initialize_config(&path).is_err()); + assert_eq!(fs::read(&path).unwrap(), b"broken existing settings"); + } + + #[test] + fn hot_folder_batch_waits_for_all_inputs_and_saves_multiple_outputs_before_deletion() { + let (_dir, config, mut task) = setup(); + task.originals = Originals::Delete; + task.settings.recognize = false; + task.settings.auto_rotate = false; + task.settings.deskew = false; + task.output_plan.steps = vec![ + output::SaveStep { grouping: output::Grouping::All, ..Default::default() }, + output::SaveStep { format: output::Format::Txt, grouping: output::Grouping::All, ..Default::default() }, + ]; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + fs::write(task.input.join("a.pdf"), text_pdf().as_slice()).unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + fs::write(task.input.join("b.pdf"), text_pdf().as_slice()).unwrap(); + step(&task, &mut state, 13, &config, None, &|| false).unwrap(); + assert!(fs::read_dir(&task.output).unwrap().next().is_none()); + step(&task, &mut state, 16, &config, None, &|| false).unwrap(); + assert!(fs::read_dir(&task.input).unwrap().next().is_none()); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 2); + let pdf = fs::read(task.output.join("Scan task.pdf")).unwrap(); + let mut session = Session::new(); + let id = session.open("combined", None, Arc::new(pdf), None).unwrap(); + assert_eq!(session.get(id).unwrap().info.pages.len(), 2); + assert_eq!(read_state(&config).unwrap().receipts.len(), 2); + } + + #[test] + fn hot_folder_failed_second_output_preserves_all_sources_and_publishes_nothing() { + let (_dir, config, mut task) = setup(); + task.originals = Originals::Delete; + task.settings.recognize = false; + task.settings.auto_rotate = false; + task.settings.deskew = false; + task.output_plan.steps.push(output::SaveStep { + format: output::Format::Txt, + folder: Some(task.archive.join("missing")), + ..Default::default() + }); + save_json(&config, &Config { tasks: vec![task.clone()] }).unwrap(); + fs::write(task.input.join("source.pdf"), text_pdf().as_slice()).unwrap(); + let mut state = State::default(); + assert!(settled(&task, &mut state, &config).is_err()); + assert!(task.input.join("source.pdf").exists()); + assert!(fs::read_dir(&task.output).unwrap().next().is_none()); + assert!(state.receipts.is_empty()); + } + + #[test] + fn hot_folder_rejects_overlapping_folders_and_invalid_options() { + let (_dir, _, mut t) = setup(); + t.output = t.input.clone(); + assert!(validate(&Config { tasks: vec![t.clone()] }, None).unwrap_err().contains("must not contain")); + t.output = t.archive.clone(); + t.settings.language = "unsupported".into(); + assert!(validate(&Config { tasks: vec![t.clone()] }, None).is_err()); + t.settings.language = "de".into(); + t.settle_seconds = 0; + assert!(validate(&Config { tasks: vec![t] }, None).is_err()); + } + + #[test] + fn hot_folder_settling_and_durable_receipts_prevent_duplicates_and_overwrites() { + if !ocr::available() { + return; + } + let (_dir, config, task) = setup(); + let original = text_pdf(); + fs::write(task.input.join("document.pdf"), original.as_slice()).unwrap(); + fs::write(task.output.join("document.pdf"), b"existing result").unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + step(&task, &mut state, 12, &config, None, &|| false).unwrap(); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 1); + step(&task, &mut state, 13, &config, None, &|| false).unwrap(); + assert_eq!(fs::read(task.output.join("document.pdf")).unwrap(), b"existing result"); + assert_eq!(fs::read(task.input.join("document.pdf")).unwrap(), *original); + let output = task.output.join("document (1).pdf"); + assert!(output.exists()); + let mut restored = read_state(&config).unwrap(); + restored.observations.clear(); + settled(&task, &mut restored, &config).unwrap(); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 2); + assert_eq!(restored.receipts.len(), 1); + } + + #[test] + fn hot_folder_changes_restart_settling_and_failures_preserve_originals() { + let (_dir, config, mut task) = setup(); + task.originals = Originals::Delete; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let source = task.input.join("broken.pdf"); + fs::write(&source, b"partial").unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + fs::write(&source, b"still not a complete PDF").unwrap(); + step(&task, &mut state, 13, &config, None, &|| false).unwrap(); + assert!(fs::read_dir(&task.output).unwrap().next().is_none()); + assert!(step(&task, &mut state, 16, &config, None, &|| false).is_err()); + assert!(source.exists()); + assert!(state.receipts.is_empty()); + } + + #[test] + fn hot_folder_move_and_delete_happen_only_after_success() { + if !ocr::available() { + return; + } + for policy in [Originals::Move, Originals::Delete] { + let (_dir, config, mut task) = setup(); + task.originals = policy; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let original = text_pdf(); + fs::write(task.input.join("source.pdf"), original.as_slice()).unwrap(); + let mut state = State::default(); + settled(&task, &mut state, &config).unwrap(); + assert!(!task.input.join("source.pdf").exists()); + assert!(task.output.join("source.pdf").exists()); + assert_eq!(read_state(&config).unwrap().receipts.len(), 1); + if policy == Originals::Move { + assert_eq!(fs::read(task.archive.join("source.pdf")).unwrap(), *original); + } + } + } + + #[test] + fn hot_folder_retry_finishes_a_journaled_archive_without_duplicate_outputs() { + let (_dir, config, mut task) = setup(); + task.originals = Originals::Move; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let original = text_pdf(); + let path = task.input.join("source.pdf"); + fs::write(&path, original.as_slice()).unwrap(); + // Simulate a crash after outputs and archive copy, before removing the original. + fs::write(task.output.join("source.pdf"), original.as_slice()).unwrap(); + fs::write(task.archive.join("source.pdf"), original.as_slice()).unwrap(); + let hash: String = Sha256::digest(original.as_slice()).iter().map(|b| format!("{b:02x}")).collect(); + let key = format!("{}:{}:{hash}", task.id, confined(&path, None).unwrap().display()); + let mut state = State::default(); + state.receipts.insert( + key.clone(), + Receipt { + outputs: vec![task.output.join("source.pdf")], + source_hash: hash, + source_stamp: fingerprint(&path).unwrap(), + output_proofs: vec![output_proof(&task.output.join("source.pdf"), None).unwrap()], + cleanup: Some(PendingLocalAction::Move { archive: confined(&task.archive, None).unwrap(), relative: "source.pdf".into() }), + }, + ); + let observation_key = format!("{}:{}", task.id, confined(&path, None).unwrap().display()); + state.observations.insert(observation_key, Observation { fingerprint: fingerprint(&path).unwrap(), since: 1, attempted: true }); + settled(&task, &mut state, &config).unwrap(); + assert!(!path.exists()); + assert_eq!(fs::read_dir(&task.archive).unwrap().count(), 1); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 1); + assert!(read_state(&config).unwrap().receipts[&key].cleanup.is_none()); + } + + #[test] + fn hot_folder_cleanup_recovery_refuses_missing_replaced_and_legacy_outputs() { + for corruption in ["missing", "replaced", "legacy"] { + let (_dir, config, mut task) = setup(); + task.originals = Originals::Delete; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let original = text_pdf(); + let source = task.input.join("source.pdf"); + let output = task.output.join("source.pdf"); + fs::write(&source, original.as_slice()).unwrap(); + fs::write(&output, original.as_slice()).unwrap(); + let proof = output_proof(&output, None).unwrap(); + let hash: String = Sha256::digest(original.as_slice()).iter().map(|b| format!("{b:02x}")).collect(); + let key = format!("{}:{}", task.id, confined(&source, None).unwrap().display()); + let stamp = fingerprint(&source).unwrap(); + let mut state = State::default(); + state.receipts.insert( + format!("{key}:{hash}"), + Receipt { + outputs: vec![output.clone()], + source_hash: hash, + source_stamp: stamp.clone(), + output_proofs: if corruption == "legacy" { vec![] } else { vec![proof] }, + cleanup: Some(PendingLocalAction::Delete), + }, + ); + state.observations.insert(key, Observation { fingerprint: stamp, since: 1, attempted: true }); + match corruption { + "missing" => fs::remove_file(&output).unwrap(), + "replaced" => fs::write(&output, b"a different result").unwrap(), + _ => {} + } + assert!(step(&task, &mut state, 10, &config, None, &|| false).is_err()); + assert_eq!(fs::read(source).unwrap(), *original); + } + } + + #[test] + fn hot_folder_replacement_during_publication_cannot_become_the_expected_output() { + let (_dir, config, mut task) = setup(); + task.originals = Originals::Delete; + task.settings.recognize = false; + task.settings.auto_rotate = false; + task.settings.deskew = false; + task.output_plan.steps.push(output::SaveStep { format: output::Format::Txt, ..Default::default() }); + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let original = text_pdf(); + let source = task.input.join("source.pdf"); + let output = task.output.join("source.pdf"); + fs::write(&source, original.as_slice()).unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + let replaced = std::cell::Cell::new(false); + let interfere = || { + if output.exists() && !replaced.replace(true) { + fs::write(&output, b"replaced before all saving steps completed").unwrap(); + } + false + }; + let error = step(&task, &mut state, 13, &config, None, &interfere).unwrap_err(); + assert!(error.contains("no longer matches"), "{error}"); + assert!(state.receipts.is_empty()); + assert_eq!(fs::read(source).unwrap(), *original); + } + + #[test] + fn hot_folder_project_recovery_verifies_every_output_before_any_acknowledgement() { + let (dir, config, mut task) = setup(); + let originals = dir.path().join("originals"); + fs::create_dir(&originals).unwrap(); + let original = text_pdf(); + for name in ["z.pdf", "a.pdf"] { + fs::write(originals.join(name), original.as_slice()).unwrap(); + } + let manifest = originals.join("project.json"); + sources::write_project_manifest( + &manifest, + &sources::ProjectManifest { files: vec!["z.pdf".into(), "a.pdf".into()], ..Default::default() }, + None, + ) + .unwrap(); + task.source = sources::SourceConfig::Project(sources::ProjectSource { manifest, action: sources::SourceAction::Delete }); + task.settings.recognize = false; + task.settings.auto_rotate = false; + task.settings.deskew = false; + task.output_plan.steps = vec![output::SaveStep { grouping: output::Grouping::All, ..Default::default() }]; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + assert!(step(&task, &mut state, 13, &config, None, &|| state_path(&config).exists()).is_err()); + assert_eq!(state.receipts.len(), 2); + let output = task.output.join("Scan task.pdf"); + let saved = fs::read(&output).unwrap(); + fs::write(&output, b"replaced output").unwrap(); + let mut restored = read_state(&config).unwrap(); + assert!(step(&task, &mut restored, 16, &config, None, &|| false).is_err()); + assert!(originals.join("z.pdf").exists() && originals.join("a.pdf").exists()); + fs::write(&output, saved).unwrap(); + step(&task, &mut restored, 19, &config, None, &|| false).unwrap(); + assert!(!originals.join("z.pdf").exists() && !originals.join("a.pdf").exists()); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 1); + } + + #[test] + fn hot_folder_pausing_or_changing_policy_during_preprocessing_leaves_the_input_retryable() { + for policy_change in [false, true] { + let (_dir, config, mut task) = setup(); + task.originals = Originals::Delete; + task.preprocessing.grayscale = true; + task.settings.recognize = false; + task.settings.auto_rotate = false; + task.settings.deskew = false; + task.settings.dpi = 72.0; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + fs::write(task.input.join("source.pdf"), text_pdf().as_slice()).unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + let calls = std::cell::Cell::new(0); + let pause = || { + calls.set(calls.get() + 1); + if calls.get() == 2 { + let mut changed = task.clone(); + if policy_change { + changed.originals = Originals::Keep; + } else { + changed.enabled = false; + } + save_config(&config, &Config { tasks: vec![changed] }, None).unwrap(); + } + false + }; + assert!(step(&task, &mut state, 13, &config, None, &pause).is_err()); + assert!(state.observations.values().all(|observation| !observation.attempted)); + assert!(task.input.join("source.pdf").exists()); + if policy_change { + task.originals = Originals::Keep; + } + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + step(&task, &mut state, 16, &config, None, &|| false).unwrap(); + assert!(task.output.join("source.pdf").exists()); + assert_eq!(task.input.join("source.pdf").exists(), policy_change); + } + } + + #[test] + fn hot_folder_recurses_and_makes_real_scans_searchable() { + if !ocr::available() { + return; + } + let (_dir, config, mut task) = setup(); + task.recursive = true; + task.settings.auto_rotate = false; + task.settings.deskew = false; + let mut s = Session::new(); + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let id = s.open("text", None, text_pdf(), None).unwrap(); + let png = crate::export::Exporter::new(s.get(id).unwrap()).png(0, 150.0).unwrap(); + fs::create_dir(task.input.join("child")).unwrap(); + fs::write(task.input.join("child/scan.png"), png).unwrap(); + let mut state = State::default(); + settled(&task, &mut state, &config).unwrap(); + let bytes = fs::read(task.output.join("child/scan.pdf")).unwrap(); + let id = s.open("result", None, Arc::new(bytes), None).unwrap(); + assert!(crate::export::Exporter::new(s.get(id).unwrap()).text(0).unwrap().contains("searchable")); + } + + #[test] + fn hot_folder_recognition_regions_exclude_new_text_without_removing_visible_content() { + if !ocr::available() { + return; + } + let (_dir, config, mut task) = setup(); + task.settings.auto_rotate = false; + task.settings.deskew = false; + task.preprocessing.regions.push(preprocess::Region { exclude: true, ..Default::default() }); + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let mut session = Session::new(); + let id = session.open("source", None, text_pdf(), None).unwrap(); + let png = crate::export::Exporter::new(session.get(id).unwrap()).png(0, 150.0).unwrap(); + fs::write(task.input.join("scan.png"), &png).unwrap(); + let mut state = State::default(); + settled(&task, &mut state, &config).unwrap(); + let bytes = fs::read(task.output.join("scan.pdf")).unwrap(); + let id = session.open("output", None, Arc::new(bytes), None).unwrap(); + let mut export = crate::export::Exporter::new(session.get(id).unwrap()); + assert!(export.text(0).unwrap().trim().is_empty()); + let rendered = image::load_from_memory(&export.png(0, 72.0).unwrap()).unwrap().to_rgb8(); + assert!(rendered.pixels().any(|p| p.0.iter().any(|v| *v < 200))); + } + + #[test] + fn hot_folder_stops_without_writing_partial_output() { + if !ocr::available() { + return; + } + let (_dir, config, task) = setup(); + fs::write(task.input.join("source.pdf"), text_pdf().as_slice()).unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + assert!(step(&task, &mut state, 13, &config, None, &|| true).unwrap_err().contains("stopped")); + assert!(fs::read_dir(&task.output).unwrap().next().is_none()); + assert!(task.input.join("source.pdf").exists()); + } + + #[test] + fn hot_folder_schedule_lock_and_root_are_enforced() { + let (dir, config, mut task) = setup(); + task.start_at = now() + 3600; + save_config(&config, &Config { tasks: vec![task] }, None).unwrap(); + assert!(poll(&config, Some(dir.path())).unwrap().tasks.is_empty()); + let lock = OpenOptions::new().create(true).truncate(false).read(true).write(true).open(config.with_extension("lock")).unwrap(); + lock.lock().unwrap(); + assert!(poll(&config, None).unwrap_err().contains("another")); + assert!(confined(&dir.path().join("../outside.pdf"), Some(dir.path())).is_err()); + } + + #[test] + fn blank_page_removal_keeps_text_and_refuses_an_empty_document() { + let s = Session::new(); + assert!(remove_blank_pages(s.create_blank(100.0, 100.0, 2).unwrap()).unwrap_err().contains("all pages")); + let text = text_pdf(); + assert_eq!(remove_blank_pages(text.clone()).unwrap(), text); + } +} diff --git a/crates/engine/src/hot_folder_analysis.rs b/crates/engine/src/hot_folder_analysis.rs new file mode 100644 index 0000000000..15db4243f5 --- /dev/null +++ b/crates/engine/src/hot_folder_analysis.rs @@ -0,0 +1,601 @@ +//! Read-only, versioned analysis of existing PDF contents. Never runs OCR, scripts, +//! rendering or saving; image rectangles are inferred from drawing instructions. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Write; +use std::sync::Arc; + +use pdfcraft_content::Matrix; +use pdfcraft_cos::{Dict, Document, ObjRef, Object}; +use serde::Serialize; + +const MAX_INPUT: usize = 512 * 1024 * 1024; +const MAX_REPORT: usize = 16 * 1024 * 1024; +const MAX_TEXT: usize = 8 * 1024 * 1024; +const MAX_CONTENT: usize = 8 * 1024 * 1024; +const MAX_STREAM: usize = 2 * 1024 * 1024; +const MAX_PAGES: usize = 10_000; +const MAX_ITEMS: usize = 50_000; +const MAX_OPERATORS: usize = 100_000; + +#[derive(Serialize)] +struct Report { + format: &'static str, + version: u32, + ocr_performed: bool, + coordinate_space: &'static str, + document: Metadata, + pages: Vec, + limitations: Vec<&'static str>, +} +#[derive(Serialize)] +struct Metadata { + pdf_version: String, + bytes: usize, + page_count: usize, + encrypted: bool, + info: BTreeMap, + has_xmp_metadata: bool, + repair_log: Vec, +} +#[derive(Serialize)] +struct PageReport { + page_index: usize, + page_number: usize, + media_box: [f64; 4], + crop_box: [f64; 4], + rotation: i64, + user_unit: f64, + display_size_points: [f64; 2], + content: ContentKind, + has_text_operators: bool, + has_vector_paint: bool, + text_blocks: Vec, + images: Vec, + warnings: Vec, +} +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +enum ContentKind { + ExistingText, + ImageWithoutText, + NoExtractableText, + EmptyContent, +} +#[derive(Serialize)] +struct TextBlock { + text: String, + rect: [f64; 4], + base_font: String, + font_size: f64, + bold: bool, + italic: bool, +} +#[derive(Serialize)] +struct Image { + rect: [f64; 4], + matrix: [f64; 6], + pixel_width: u32, + pixel_height: u32, + resource: Option, + inline: bool, +} + +/// Analyze zero-based pages, preserving their requested order; an empty selection +/// means every page. Duplicate or nonexistent pages are errors. Output is UTF-8 JSON +/// version 1, at most 16 MiB. Input bytes and originals are never modified. +/// This synchronous API belongs on a worker thread; cancellation is checked between +/// content operators, form calls and extraction phases. No OCR model is consulted. +pub fn analyze(bytes: &[u8], pages: &[usize], should_stop: &dyn Fn() -> bool) -> Result, String> { + crate::guard(|| analyze_inner(bytes, pages, should_stop))? +} + +fn analyze_inner(bytes: &[u8], selection: &[usize], stop: &dyn Fn() -> bool) -> Result, String> { + check(stop)?; + if bytes.is_empty() || bytes.len() > MAX_INPUT { + return Err("analysis needs a PDF of at most 512 MiB".into()); + } + let doc = Document::open_with_stream_limit(Arc::new(bytes.to_vec()), None, MAX_STREAM).map_err(|error| format!("cannot analyze PDF: {error}"))?; + if doc.permissions().is_some_and(|permissions| !permissions.copy()) { + return Err("PDF permissions do not allow text extraction".into()); + } + check(stop)?; + let all = pdfcraft_model::pages(&doc); + if all.is_empty() || all.len() > MAX_PAGES || selection.len() > MAX_PAGES { + return Err("analysis supports 1..10000 pages".into()); + } + let selected = if selection.is_empty() { (0..all.len()).collect::>() } else { selection.to_vec() }; + let mut seen = BTreeSet::new(); + if selected.iter().any(|page| *page >= all.len() || !seen.insert(*page)) { + return Err("analysis selection contains a duplicate or nonexistent page".into()); + } + let mut budget = Budget::default(); + let mut info = BTreeMap::new(); + for key in ["Title", "Author", "Subject", "Keywords", "Creator", "Producer", "CreationDate", "ModDate", "Trapped"] { + if let Some(value) = pdfcraft_organize::info(&doc, key) { + budget.text(&value)?; + info.insert(key.into(), value); + } + } + let repair_log = doc.repair_log().to_vec(); + for warning in &repair_log { + budget.text(warning)?; + } + let has_xmp_metadata = doc.root().and_then(|root| doc.get(root).as_dict().map(|dict| dict.contains(b"Metadata"))).unwrap_or(false); + let document = Metadata { + pdf_version: doc.version().into(), + bytes: bytes.len(), + page_count: all.len(), + encrypted: doc.security().is_some(), + info, + has_xmp_metadata, + repair_log, + }; + let mut reports = Vec::with_capacity(selected.len()); + for page_index in selected { + check(stop)?; + let page = all.get(page_index).ok_or("analysis page is missing")?; + let crop_box = page.crop(&doc); + let media_box = rectangle(&doc, page.dict.get(b"MediaBox"))?.unwrap_or([0.0, 0.0, 612.0, 792.0]); + let (width, height) = page.display_size(&doc); + if ![width, height].iter().all(|value| value.is_finite() && *value > 0.0) { + return Err(format!("page {} has invalid display geometry", page_index + 1)); + } + let mut inventory = Inventory::default(); + let resources = dictionary(&doc, page.dict.get(b"Resources")); + let mut state = Graphics::default(); + let mut work = Work::default(); + if let Some(contents) = page.dict.get(b"Contents") { + let resolved = doc.resolve(contents); + let streams = if let Object::Array(array) = &*resolved { array.clone() } else { vec![contents.clone()] }; + if streams.len() > 4096 { + return Err("analysis page has too many content streams".into()); + } + for object in streams { + let object = doc.resolve(&object); + let Object::Stream(stream) = &*object else { + return Err("analysis page has a malformed content stream".into()); + }; + let data = stream.decoded_within(MAX_STREAM).map_err(|error| format!("analysis content cannot be decoded: {error}"))?; + walk(&doc, &data, &resources, &mut state, &mut inventory, &mut work, stop)?; + } + } + check(stop)?; + let blocks = + pdfcraft_edit::reading_blocks(&doc, page_index).map_err(|error| format!("cannot analyze text on page {}: {error}", page_index + 1))?; + if blocks.len().saturating_add(inventory.images.len()) > MAX_ITEMS { + return Err("analysis page exceeds 50000 text/image regions".into()); + } + let mut text_blocks = Vec::new(); + for block in blocks { + check(stop)?; + if block.text.trim().is_empty() { + continue; + } + if !block.rect.iter().chain([&block.size]).all(|value| value.is_finite()) { + return Err("analysis encountered non-finite text geometry".into()); + } + budget.text(&block.text)?; + budget.text(&block.base_font)?; + text_blocks.push(TextBlock { + text: block.text, + rect: block.rect, + base_font: block.base_font, + font_size: block.size, + bold: block.bold, + italic: block.italic, + }); + } + for image in &inventory.images { + if let Some(resource) = &image.resource { + budget.text(resource)?; + } + } + budget.items = budget.items.saturating_add(text_blocks.len()).saturating_add(inventory.images.len()); + if budget.items > MAX_ITEMS { + return Err("analysis report exceeds 50000 text/image regions; select fewer pages".into()); + } + let content = if !text_blocks.is_empty() { + ContentKind::ExistingText + } else if !inventory.images.is_empty() { + ContentKind::ImageWithoutText + } else if inventory.has_text || inventory.has_vector || inventory.other_paint { + ContentKind::NoExtractableText + } else { + ContentKind::EmptyContent + }; + if inventory.has_text && text_blocks.is_empty() { + inventory.warnings.push("Text drawing operators exist, but their text could not be decoded; OCR was not run.".into()); + } + if page.dict.contains(b"Annots") { + inventory.warnings.push("Annotation and form-widget appearances are not part of this content inventory.".into()); + } + reports.push(PageReport { + page_index, + page_number: page_index + 1, + media_box, + crop_box, + rotation: page.rotation(&doc), + user_unit: page.user_unit(&doc), + display_size_points: [width, height], + content, + has_text_operators: inventory.has_text, + has_vector_paint: inventory.has_vector, + text_blocks, + images: inventory.images, + warnings: inventory.warnings, + }); + } + check(stop)?; + let report = Report { + format: "pdfcraft-analysis", + version: 1, + ocr_performed: false, + coordinate_space: "PDF default user space; unrotated, y upward. Multiply by page.user_unit for points. Rectangles are [x0,y0,x1,y1].", + document, + pages: reports, + limitations: vec![ + "This describes existing PDF instructions and decoded text, not visual OCR or semantic layout recognition.", + "Text blocks retain content order; reading order in complex layouts is not guaranteed.", + "Image rectangles are transformed placement bounds; clipping, masks, transparency and optional-layer visibility are not applied.", + "An image without text may be a scan or other artwork; an empty content inventory does not prove the rendered page is blank.", + "Document Info fields are reported; XMP presence is reported without parsing its separate metadata packet.", + ], + }; + let mut output = LimitedOutput(Vec::new()); + serde_json::to_writer_pretty(&mut output, &report).map_err(|_| "analysis report exceeds 16 MiB or could not be encoded")?; + check(stop)?; + Ok(output.0) +} + +#[derive(Default)] +struct Budget { + text: usize, + items: usize, +} +impl Budget { + fn text(&mut self, text: &str) -> Result<(), String> { + self.text = self.text.saturating_add(text.len()); + if self.text > MAX_TEXT { Err("analysis text exceeds 8 MiB; select fewer pages".into()) } else { Ok(()) } + } +} +struct LimitedOutput(Vec); +impl Write for LimitedOutput { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.0.len().saturating_add(bytes.len()) > MAX_REPORT { + return Err(std::io::Error::other("analysis output limit")); + } + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +fn check(stop: &dyn Fn() -> bool) -> Result<(), String> { + if stop() { Err("analysis cancelled; originals unchanged".into()) } else { Ok(()) } +} +fn dictionary(doc: &Document, object: Option<&Object>) -> Dict { + object.map(|value| doc.resolve(value)).and_then(|value| value.as_dict().cloned()).unwrap_or_default() +} +fn rectangle(doc: &Document, object: Option<&Object>) -> Result, String> { + let Some(object) = object else { + return Ok(None); + }; + let object = doc.resolve(object); + let array = object.as_array().filter(|array| array.len() == 4).ok_or("analysis encountered an invalid rectangle")?; + let mut values = [0.0_f64; 4]; + for (target, value) in values.iter_mut().zip(array) { + *target = doc.resolve(value).as_f64().filter(|value| value.is_finite()).ok_or("analysis rectangle is non-finite")?; + } + Ok(Some([values[0].min(values[2]), values[1].min(values[3]), values[0].max(values[2]), values[1].max(values[3])])) +} +#[derive(Default)] +struct Inventory { + images: Vec, + has_text: bool, + has_vector: bool, + other_paint: bool, + warnings: Vec, +} +struct Graphics { + matrix: Matrix, + stack: Vec, +} +impl Default for Graphics { + fn default() -> Self { + Self { matrix: Matrix::IDENTITY, stack: Vec::new() } + } +} +#[derive(Default)] +struct Work { + decoded: usize, + operators: usize, + forms: usize, + depth: usize, + path: Vec, +} + +#[allow(clippy::too_many_arguments)] +fn walk( + doc: &Document, + data: &[u8], + resources: &Dict, + graphics: &mut Graphics, + inventory: &mut Inventory, + work: &mut Work, + stop: &dyn Fn() -> bool, +) -> Result<(), String> { + check(stop)?; + work.decoded = work.decoded.saturating_add(data.len()); + if work.decoded > MAX_CONTENT { + return Err("analysis page expands beyond 8 MiB of content".into()); + } + let parsed = pdfcraft_content::parse(data); + if parsed.skipped > 0 { + return Err("analysis encountered malformed content instructions; report was not published".into()); + } + work.operators = work.operators.saturating_add(parsed.ops.len()); + if work.operators > MAX_OPERATORS { + return Err("analysis page exceeds 100000 drawing instructions".into()); + } + let xobjects = dictionary(doc, resources.get(b"XObject")); + for op in parsed.ops { + check(stop)?; + match op.op.as_slice() { + b"q" => { + if graphics.stack.len() >= 256 { + return Err("analysis graphics-state nesting exceeds 256".into()); + } + graphics.stack.push(graphics.matrix); + } + b"Q" => graphics.matrix = graphics.stack.pop().ok_or("analysis encountered an unbalanced graphics state")?, + b"cm" => { + let matrix = + op.nums::<6>().filter(|matrix| matrix.iter().all(|value| value.is_finite())).ok_or("analysis encountered an invalid matrix")?; + graphics.matrix = Matrix(matrix).then(&graphics.matrix); + if !graphics.matrix.0.iter().all(|value| value.is_finite()) { + return Err("analysis matrix overflow".into()); + } + } + b"Tj" | b"TJ" | b"'" | b"\"" => inventory.has_text = true, + b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" => inventory.has_vector = true, + b"sh" => inventory.other_paint = true, + b"BI" => { + let (dict, _) = op.inline.as_ref().ok_or("analysis inline image is malformed")?; + push_image(dict, graphics.matrix, None, true, inventory)?; + } + b"Do" => { + let name = op.name(0).ok_or("analysis XObject call has no name")?; + let object = xobjects.get(name).ok_or("analysis XObject resource is missing")?; + let id = object.as_ref(); + let object = doc.resolve(object); + let Object::Stream(stream) = &*object else { + return Err("analysis XObject resource is not a stream".into()); + }; + match stream.dict.name(b"Subtype") { + Some(b"Image") => push_image(&stream.dict, graphics.matrix, Some(String::from_utf8_lossy(name).into_owned()), false, inventory)?, + Some(b"Form") => { + work.forms = work.forms.saturating_add(1); + if work.depth >= 12 || work.forms > 256 || id.is_some_and(|id| work.path.contains(&id)) { + return Err("analysis form recursion or visit limit reached".into()); + } + let form_matrix = if let Some(matrix) = stream.dict.get(b"Matrix") { + let value = doc.resolve(matrix); + value.as_array().and_then(|values| Matrix::from_operands(values)).ok_or("analysis form matrix is malformed")? + } else { + Matrix::IDENTITY + }; + let matrix = form_matrix.then(&graphics.matrix); + if !matrix.0.iter().all(|value| value.is_finite()) { + return Err("analysis form matrix is non-finite".into()); + } + let form_resources = + if stream.dict.contains(b"Resources") { dictionary(doc, stream.dict.get(b"Resources")) } else { resources.clone() }; + let data = stream.decoded_within(MAX_STREAM).map_err(|error| format!("analysis form cannot be decoded: {error}"))?; + if let Some(id) = id { + work.path.push(id); + } + work.depth += 1; + let mut inner = Graphics { matrix, stack: Vec::new() }; + walk(doc, &data, &form_resources, &mut inner, inventory, work, stop)?; + work.depth -= 1; + if id.is_some() { + work.path.pop(); + } + } + _ => { + inventory.other_paint = true; + if inventory.warnings.is_empty() { + inventory.warnings.push("An unsupported XObject subtype is present.".into()); + } + } + } + } + _ => {} + } + } + Ok(()) +} +fn push_image(dict: &Dict, matrix: Matrix, resource: Option, inline: bool, inventory: &mut Inventory) -> Result<(), String> { + if inventory.images.len() >= MAX_ITEMS { + return Err("analysis page exceeds 50000 image placements".into()); + } + let rect = matrix.bbox([0.0, 0.0, 1.0, 1.0]); + if !rect.iter().all(|value| value.is_finite()) { + return Err("analysis image bounds are non-finite".into()); + } + let dimension = |long, short| { + dict.int(long) + .or_else(|| dict.int(short)) + .and_then(|value| u32::try_from(value).ok()) + .filter(|value| *value > 0) + .ok_or_else(|| "analysis image dimensions are missing or invalid".to_owned()) + }; + inventory.images.push(Image { + rect, + matrix: matrix.0, + pixel_width: dimension(b"Width", b"W")?, + pixel_height: dimension(b"Height", b"H")?, + resource, + inline, + }); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use pdfcraft_cos::{SaveOptions, Stream, write_full}; + use serde_json::{Value, json}; + + fn nums(values: &[f64]) -> Object { + Object::Array(values.iter().map(|value| Object::Real(*value)).collect()) + } + fn fixture() -> Document { + let mut doc = pdfcraft_create::blank(200.0, 300.0, 3).unwrap(); + let pages = pdfcraft_model::pages(&doc); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type1")); + font.set(b"BaseFont".to_vec(), Object::name("Helvetica")); + let font = doc.add(font); + let mut fonts = Dict::new(); + fonts.set(b"F1".to_vec(), Object::Ref(font)); + let mut resources = Dict::new(); + resources.set(b"Font".to_vec(), fonts); + let text = doc.add(Stream::from_raw(Dict::new(), b"BT /F1 12 Tf 20 230 Td (Existing searchable text) Tj ET".to_vec())); + doc.update_dict(pages[0].obj, |dict| { + dict.set(b"Resources".to_vec(), resources); + dict.set(b"Contents".to_vec(), Object::Ref(text)); + dict.set(b"CropBox".to_vec(), nums(&[10.0, 20.0, 190.0, 290.0])); + dict.set(b"Rotate".to_vec(), Object::Int(90)); + dict.set(b"UserUnit".to_vec(), Object::Real(2.0)); + }) + .unwrap(); + let mut image = Dict::new(); + image.set(b"Subtype".to_vec(), Object::name("Image")); + image.set(b"Width".to_vec(), Object::Int(1)); + image.set(b"Height".to_vec(), Object::Int(1)); + image.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + image.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + let image = doc.add(Stream::from_raw(image, vec![90, 100, 110])); + let mut objects = Dict::new(); + objects.set(b"Im".to_vec(), Object::Ref(image)); + let mut inner_res = Dict::new(); + inner_res.set(b"XObject".to_vec(), objects); + let mut form = Dict::new(); + form.set(b"Subtype".to_vec(), Object::name("Form")); + form.set(b"BBox".to_vec(), nums(&[0.0, 0.0, 200.0, 300.0])); + form.set(b"Matrix".to_vec(), nums(&[1.0, 0.0, 0.0, 1.0, 5.0, 7.0])); + form.set(b"Resources".to_vec(), inner_res); + let form = doc.add(Stream::from_raw(form, b"q 20 0 0 30 1 2 cm /Im Do Q".to_vec())); + let mut objects = Dict::new(); + objects.set(b"Fm".to_vec(), Object::Ref(form)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), objects); + let content = doc.add(Stream::from_raw( + Dict::new(), + b"q 2 0 0 2 50 60 cm /Fm Do Q q 10 0 0 5 100 100 cm BI /W 1 /H 1 /CS /RGB /BPC 8 ID abc EI Q".to_vec(), + )); + doc.update_dict(pages[1].obj, |dict| { + dict.set(b"Resources".to_vec(), resources); + dict.set(b"Contents".to_vec(), Object::Ref(content)); + }) + .unwrap(); + pdfcraft_organize::set_info(&mut doc, "Title", "Analyse ä <&> 日本語").unwrap(); + pdfcraft_organize::set_info(&mut doc, "CreationDate", "D:20261011120000+02'00'").unwrap(); + doc + } + fn bytes(doc: &Document) -> Vec { + write_full(doc, &SaveOptions::default()).unwrap() + } + + #[test] + fn analysis_reports_text_scan_empty_metadata_and_nested_image_geometry_without_mutation() { + let pdf = bytes(&fixture()); + let before = pdf.clone(); + let report = analyze(&pdf, &[], &|| false).unwrap(); + assert_eq!(report, analyze(&pdf, &[], &|| false).unwrap(), "report must be deterministic"); + assert_eq!(pdf, before); + let report: Value = serde_json::from_slice(&report).unwrap(); + assert_eq!(report["format"], "pdfcraft-analysis"); + assert_eq!(report["version"], 1); + assert_eq!(report["ocr_performed"], false); + assert_eq!(report["document"]["info"]["Title"], "Analyse ä <&> 日本語"); + assert_eq!(report["document"]["info"]["CreationDate"], "D:20261011120000+02'00'"); + let pages = report["pages"].as_array().unwrap(); + assert_eq!(pages[0]["content"], "existing_text"); + assert_eq!(pages[0]["text_blocks"][0]["text"], "Existing searchable text"); + assert_eq!(pages[0]["crop_box"], json!([10.0, 20.0, 190.0, 290.0])); + assert_eq!(pages[0]["display_size_points"], json!([540.0, 360.0])); + assert_eq!(pages[0]["rotation"], 90); + assert_eq!(pages[0]["user_unit"], 2.0); + assert_eq!(pages[1]["content"], "image_without_text"); + assert_eq!(pages[1]["images"][0]["rect"], json!([62.0, 78.0, 102.0, 138.0])); + assert_eq!(pages[1]["images"][1]["rect"], json!([100.0, 100.0, 110.0, 105.0])); + assert_eq!(pages[1]["images"][1]["inline"], true); + assert_eq!(pages[2]["content"], "empty_content"); + } + + #[test] + fn analysis_preserves_selection_order_and_refuses_bad_selection_malformed_pdf_and_cancel() { + let pdf = bytes(&fixture()); + let report: Value = serde_json::from_slice(&analyze(&pdf, &[2, 0], &|| false).unwrap()).unwrap(); + assert_eq!(report["pages"][0]["page_number"], 3); + assert_eq!(report["pages"][1]["page_number"], 1); + assert!(analyze(&pdf, &[3], &|| false).is_err()); + assert!(analyze(&pdf, &[1, 1], &|| false).is_err()); + assert!(analyze(b"not a PDF", &[], &|| false).is_err()); + assert!(analyze(&pdf, &[], &|| true).unwrap_err().contains("cancelled")); + let calls = std::cell::Cell::new(0); + assert!( + analyze(&pdf, &[], &|| { + calls.set(calls.get() + 1); + calls.get() > 10 + }) + .unwrap_err() + .contains("cancelled") + ); + } + + #[test] + fn analysis_rejects_recursive_forms_and_oversized_content_without_truncating_report() { + let mut doc = pdfcraft_create::blank(200.0, 300.0, 1).unwrap(); + let page = pdfcraft_model::pages(&doc)[0].obj; + let form = doc.add(Object::Null); + let mut objects = Dict::new(); + objects.set(b"Fm".to_vec(), Object::Ref(form)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), objects); + let mut form_dict = Dict::new(); + form_dict.set(b"Subtype".to_vec(), Object::name("Form")); + form_dict.set(b"BBox".to_vec(), nums(&[0.0, 0.0, 200.0, 300.0])); + form_dict.set(b"Resources".to_vec(), resources.clone()); + doc.set(form, Stream::from_raw(form_dict, b"/Fm Do".to_vec())); + let content = doc.add(Stream::from_raw(Dict::new(), b"/Fm Do".to_vec())); + doc.update_dict(page, |dict| { + dict.set(b"Contents".to_vec(), Object::Ref(content)); + dict.set(b"Resources".to_vec(), resources); + }) + .unwrap(); + assert!(analyze(&bytes(&doc), &[], &|| false).unwrap_err().contains("recursion")); + doc.set(content, Stream::from_raw(Dict::new(), vec![b' '; MAX_STREAM + 1])); + assert!(analyze(&bytes(&doc), &[], &|| false).is_err()); + } + + #[test] + fn analysis_does_not_claim_vector_art_or_undecodable_text_is_a_blank_scan() { + let mut doc = pdfcraft_create::blank(200.0, 300.0, 1).unwrap(); + let page = pdfcraft_model::pages(&doc)[0].obj; + let content = doc.add(Stream::from_raw(Dict::new(), b"10 10 50 50 re f".to_vec())); + doc.update_dict(page, |dict| dict.set(b"Contents".to_vec(), Object::Ref(content))).unwrap(); + let report: Value = serde_json::from_slice(&analyze(&bytes(&doc), &[], &|| false).unwrap()).unwrap(); + assert_eq!(report["pages"][0]["content"], "no_extractable_text"); + assert_eq!(report["pages"][0]["has_vector_paint"], true); + doc.set(content, Stream::from_raw(Dict::new(), b"BT /MissingFont 12 Tf (text) Tj ET".to_vec())); + let report: Value = serde_json::from_slice(&analyze(&bytes(&doc), &[], &|| false).unwrap()).unwrap(); + assert_ne!(report["pages"][0]["content"], "empty_content"); + assert_eq!(report["pages"][0]["has_text_operators"], true); + } +} diff --git a/crates/engine/src/hot_folder_output.rs b/crates/engine/src/hot_folder_output.rs new file mode 100644 index 0000000000..3f74aef48f --- /dev/null +++ b/crates/engine/src/hot_folder_output.rs @@ -0,0 +1,735 @@ +//! Hot Folder output planning and conversion. Sources are never modified here. +//! +//! Every conversion is staged before publishing any file. Each publication uses +//! atomic no-clobber persistence. Since different destinations can be on different +//! volumes, a publication failure reports the complete files already saved; only a +//! successful receipt authorizes the caller's configured source move/delete step. + +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::export::{Exporter, ImageFormat}; +use crate::{Session, hot_folder::confined}; + +const MAX_INPUT_BYTES: u64 = 512 * 1024 * 1024; +const MAX_STAGED_BYTES: u64 = 1024 * 1024 * 1024; +const MAX_OUTPUT_FILES: usize = 10_000; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Grouping { + #[default] + File, + Folder, + All, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Format { + /// Keep the processed PDF, including its OCR text if recognition was enabled. + #[default] + Pdf, + /// Render all visible content; create an image-only PDF without hidden OCR text. + ImagePdf, + Docx, + Html, + Rtf, + Xlsx, + Epub, + Analysis, + Txt, + Png, + Jpeg, + Tiff, +} + +impl Format { + pub const ALL: &[Self] = &[ + Self::Pdf, + Self::ImagePdf, + Self::Docx, + Self::Html, + Self::Rtf, + Self::Xlsx, + Self::Epub, + Self::Txt, + Self::Png, + Self::Jpeg, + Self::Tiff, + Self::Analysis, + ]; + pub fn extension(self) -> &'static str { + match self { + Self::Pdf | Self::ImagePdf => "pdf", + Self::Docx => "docx", + Self::Html => "html", + Self::Rtf => "rtf", + Self::Xlsx => "xlsx", + Self::Epub => "epub", + Self::Analysis => "json", + Self::Txt => "txt", + Self::Png => "png", + Self::Jpeg => "jpg", + Self::Tiff => "tif", + } + } + pub fn label(self) -> &'static str { + match self { + Self::Pdf => "PDF (processed document)", + Self::ImagePdf => "PDF (image only)", + Self::Docx => "Word (DOCX)", + Self::Html => "HTML", + Self::Rtf => "Rich text (RTF)", + Self::Xlsx => "Excel (XLSX)", + Self::Epub => "E-book (EPUB)", + Self::Analysis => "Layout report (JSON)", + Self::Txt => "Plain text (TXT)", + Self::Png => "PNG images", + Self::Jpeg => "JPEG images", + Self::Tiff => "TIFF images", + } + } + fn image(self, quality: u8) -> Option { + match self { + Self::Png => Some(ImageFormat::Png), + Self::Jpeg => Some(ImageFormat::Jpeg { quality }), + Self::Tiff => Some(ImageFormat::Tiff), + _ => None, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct SaveStep { + /// None uses the task's output directory. Explicit folders must already exist. + pub folder: Option, + pub format: Format, + pub grouping: Grouping, + /// File stem, without extension. Tokens: {name}, {folder}, {task}, {date}, {time}, {counter}. + /// Date/time use the batch's UTC timestamp, so all steps agree. Counter is one-based. + pub name_template: String, + pub dpi: f64, + pub jpeg_quality: u8, + /// EPUB language tag; "und" means undetermined. The scheduler fills its OCR language. + pub language: String, +} + +impl Default for SaveStep { + fn default() -> Self { + Self { + folder: None, + format: Format::Pdf, + grouping: Grouping::File, + name_template: "{name}".into(), + dpi: 150.0, + jpeg_quality: 90, + language: "und".into(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct OutputPlan { + pub steps: Vec, +} + +impl Default for OutputPlan { + fn default() -> Self { + Self { steps: vec![SaveStep::default()] } + } +} + +impl OutputPlan { + pub fn needs_batch(&self) -> bool { + self.steps.iter().any(|step| step.grouping != Grouping::File) + } + + /// Validate before reading source files. The caller also checks destinations against + /// its full set of input folders to prevent recursive output/input loops. + pub fn validate(&self, default_output: &Path, root: Option<&Path>) -> Result<(), String> { + if self.steps.is_empty() || self.steps.len() > 16 { + return Err("choose between 1 and 16 output saving steps".into()); + } + for step in &self.steps { + if !step.dpi.is_finite() || !(18.0..=1200.0).contains(&step.dpi) || !(1..=100).contains(&step.jpeg_quality) { + return Err("output resolution must be 18..1200 dpi and JPEG quality 1..100".into()); + } + validate_template(&step.name_template)?; + if step.language.is_empty() || step.language.len() > 63 || !step.language.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') { + return Err("EPUB language must be a language tag such as de-AT or und".into()); + } + let folder = confined(step.folder.as_deref().unwrap_or(default_output), root)?; + if !folder.is_dir() { + return Err(format!("output folder does not exist: {}", folder.display())); + } + } + Ok(()) + } +} + +/// An already processed PDF and its source-relative path; pages stay in this input order. +#[derive(Clone, Debug)] +pub struct Input { + pub relative: PathBuf, + pub bytes: Arc>, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct OutputReceipt { + pub outputs: Vec, + pub proofs: Vec, + /// Rendered pages that were bounded by the renderer's pixel limit. + pub warnings: Vec, +} + +/// Expected content computed from staged export bytes, before publication. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct OutputProof { + pub path: PathBuf, + pub bytes: u64, + pub sha256: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct OutputError { + pub message: String, + /// Complete files already published (or reused after byte-identical comparison). + /// No source action is permitted after this error, even when this list is nonempty. + pub outputs: Vec, +} + +impl std::fmt::Display for OutputError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}; all originals retained", self.message) + } +} +impl std::error::Error for OutputError {} + +/// Blocking worker API. An error always keeps sources untouched. Original-file actions +/// belong to the scheduler only after it durably journals the complete successful receipt. +pub fn save( + inputs: &[Input], + default_output: &Path, + plan: &OutputPlan, + task_name: &str, + batch_time: u64, + root: Option<&Path>, + should_stop: &dyn Fn() -> bool, +) -> Result { + let mut receipt = OutputReceipt::default(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + save_inner(inputs, default_output, plan, task_name, batch_time, root, should_stop, &mut receipt) + })); + match result { + Ok(Ok(())) => Ok(receipt), + Ok(Err(message)) => Err(OutputError { message, outputs: receipt.outputs }), + Err(_) => Err(OutputError { message: "output conversion failed unexpectedly".into(), outputs: receipt.outputs }), + } +} + +struct Group<'a> { + inputs: Vec<&'a Input>, + subfolder: PathBuf, + name: String, + folder_name: String, +} + +fn relative_ok(relative: &Path) -> bool { + !relative.as_os_str().is_empty() && relative.components().all(|part| matches!(part, Component::Normal(_))) +} + +fn groups<'a>(inputs: &'a [Input], grouping: Grouping, task_name: &str) -> Vec> { + let folder_name = |parent: &Path| parent.file_name().map_or_else(|| task_name.to_owned(), |name| name.to_string_lossy().into_owned()); + match grouping { + Grouping::File => inputs + .iter() + .map(|input| { + let parent = input.relative.parent().unwrap_or(Path::new("")); + Group { + inputs: vec![input], + subfolder: parent.to_owned(), + name: input.relative.file_stem().unwrap_or_default().to_string_lossy().into_owned(), + folder_name: folder_name(parent), + } + }) + .collect(), + Grouping::Folder => { + let mut by_folder: BTreeMap> = BTreeMap::new(); + for input in inputs { + by_folder.entry(input.relative.parent().unwrap_or(Path::new("")).to_owned()).or_default().push(input); + } + by_folder + .into_iter() + .map(|(subfolder, inputs)| Group { name: folder_name(&subfolder), folder_name: folder_name(&subfolder), subfolder, inputs }) + .collect() + } + Grouping::All => { + vec![Group { inputs: inputs.iter().collect(), subfolder: PathBuf::new(), name: task_name.into(), folder_name: task_name.into() }] + } + } +} + +struct Staged { + file: tempfile::NamedTempFile, + folder: PathBuf, + stem: String, + extension: &'static str, + size: u64, + sha256: String, +} + +#[allow(clippy::too_many_arguments)] +fn save_inner( + inputs: &[Input], + default_output: &Path, + plan: &OutputPlan, + task_name: &str, + batch_time: u64, + root: Option<&Path>, + should_stop: &dyn Fn() -> bool, + receipt: &mut OutputReceipt, +) -> Result<(), String> { + plan.validate(default_output, root)?; + if inputs.is_empty() || inputs.len() > 10_000 || inputs.iter().any(|input| !relative_ok(&input.relative)) { + return Err("output batch needs 1..10000 inputs with relative paths containing normal components only".into()); + } + let mut total = 0_u64; + for input in inputs { + total = total.saturating_add(input.bytes.len() as u64); + if total > MAX_INPUT_BYTES { + return Err("combined output inputs exceed 512 MiB; use smaller batches".into()); + } + } + let timestamp = i64::try_from(batch_time).ok().and_then(|value| DateTime::::from_timestamp(value, 0)).ok_or("invalid output timestamp")?; + let mut staged = Vec::new(); + let mut staged_bytes = 0_u64; + for step in &plan.steps { + let base = confined(step.folder.as_deref().unwrap_or(default_output), root)?; + for (index, group) in groups(inputs, step.grouping, task_name).into_iter().enumerate() { + stopped(should_stop)?; + let folder = safe_subfolder(&base, &group.subfolder)?; + let stem = expand_name(&step.name_template, &group.name, &group.folder_name, task_name, timestamp, index.saturating_add(1))?; + let bytes = if let [input] = group.inputs.as_slice() { + input.bytes.clone() + } else { + let sources = + group.inputs.iter().map(|input| (input.relative.to_string_lossy().into_owned(), input.bytes.clone())).collect::>(); + Session::new().combine(&sources).map_err(|error| error.to_string())? + }; + if step.format == Format::Analysis { + let report = crate::hot_folder_analysis::analyze(bytes.as_slice(), &[], should_stop)?; + stage(&mut staged, &mut staged_bytes, &folder, &stem, "json", &report)?; + continue; + } + let mut session = Session::new(); + let id = session.open(format!("{stem}.pdf"), None, bytes.clone(), None).map_err(|error| error.to_string())?; + let document = session.get(id).ok_or("output document unavailable")?; + let pages = document.info.pages.len(); + if pages == 0 || pages > MAX_OUTPUT_FILES { + return Err("output document needs 1..10000 pages".into()); + } + let mut exporter = Exporter::new(document); + if let Some(format) = step.format.image(step.jpeg_quality) { + for page in 0..pages { + stopped(should_stop)?; + warn_resolution(&exporter, page, step.dpi, &stem, receipt); + let bytes = exporter.image(page, step.dpi, format)?; + stage( + &mut staged, + &mut staged_bytes, + &folder, + &format!("{stem}_page_{:04}", page.saturating_add(1)), + step.format.extension(), + &bytes, + )?; + } + } else { + let output = match step.format { + Format::Pdf => bytes, + Format::ImagePdf => { + let mut sources = Vec::new(); + let mut images_size = 0_u64; + for page in 0..pages { + stopped(should_stop)?; + warn_resolution(&exporter, page, step.dpi, &stem, receipt); + let png = exporter.png(page, step.dpi)?; + images_size = images_size.saturating_add(png.len() as u64); + if images_size > MAX_INPUT_BYTES { + return Err("image-only PDF exceeds 512 MiB; lower the resolution".into()); + } + let name = format!("page-{}.png", page.saturating_add(1)); + let pdf = session + .create_from_images_with_resolution( + &[(name.clone(), png)], + crate::ImageResolution::Dpi(exporter.dpi_used(page, step.dpi)), + ) + .map_err(|error| error.to_string())?; + sources.push((format!("page-{}.pdf", page.saturating_add(1)), pdf)); + } + session.combine(&sources).map_err(|error| error.to_string())? + } + Format::Txt => Arc::new(exporter.text_of(&(0..pages).collect::>())?.into_bytes()), + Format::Docx => Arc::new(document.export_office(crate::compare::OfficeFormat::Docx)), + Format::Html => Arc::new(document.export_office(crate::compare::OfficeFormat::Html)), + Format::Rtf => Arc::new(document.export_office(crate::compare::OfficeFormat::Rtf)), + Format::Xlsx => Arc::new(pdfcraft_export::xlsx(&document.export_pages(), &stem)?), + Format::Epub => Arc::new(pdfcraft_export::epub(&document.export_pages(), &stem, &step.language)?), + Format::Png | Format::Jpeg | Format::Tiff | Format::Analysis => return Err("invalid output state".into()), + }; + stage(&mut staged, &mut staged_bytes, &folder, &stem, step.format.extension(), output.as_slice())?; + } + } + } + // All render/export work succeeded; only now publish the staged files. + for file in staged { + stopped(should_stop)?; + let bytes = file.size; + let sha256 = file.sha256.clone(); + let path = publish(file, root)?; + receipt.proofs.push(OutputProof { path: path.clone(), bytes, sha256 }); + receipt.outputs.push(path); + } + Ok(()) +} + +fn warn_resolution(exporter: &Exporter, page: usize, dpi: f64, name: &str, receipt: &mut OutputReceipt) { + let actual = exporter.dpi_used(page, dpi); + if actual + 0.1 < dpi { + receipt.warnings.push(format!( + "{name}, page {}: rendered at {actual:.1} dpi instead of {dpi:.1} because of the page pixel limit", + page.saturating_add(1) + )); + } +} + +fn stopped(should_stop: &dyn Fn() -> bool) -> Result<(), String> { + if should_stop() { Err("output task stopped".into()) } else { Ok(()) } +} + +fn digest(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn stage(staged: &mut Vec, total: &mut u64, folder: &Path, stem: &str, extension: &'static str, bytes: &[u8]) -> Result<(), String> { + *total = total.saturating_add(bytes.len() as u64); + if bytes.is_empty() || *total > MAX_STAGED_BYTES || staged.len() >= MAX_OUTPUT_FILES { + return Err("output batch exceeds 1 GiB or 10000 files, or an exporter returned an empty file; use smaller batches".into()); + } + let mut file = tempfile::Builder::new().prefix(".pdfcraft-output-").tempfile_in(folder).map_err(|error| error.to_string())?; + file.write_all(bytes).and_then(|_| file.as_file().sync_all()).map_err(|error| format!("cannot stage output: {error}"))?; + staged.push(Staged { file, folder: folder.to_owned(), stem: stem.to_owned(), extension, size: bytes.len() as u64, sha256: digest(bytes) }); + Ok(()) +} + +fn publish(mut file: Staged, root: Option<&Path>) -> Result { + if confined(&file.folder, root)? != file.folder { + return Err("output directory changed during conversion".into()); + } + for counter in 0..10_000 { + let name = if counter == 0 { format!("{}.{}", file.stem, file.extension) } else { format!("{} ({counter}).{}", file.stem, file.extension) }; + let target = file.folder.join(name); + // Identical complete outputs are reusable on retry; different data always gets a + // suffix. Links/directories are never followed or overwritten. + if same_file(&target, file.size, &file.sha256)? { + crate::hot_folder::sync_parent(&target)?; + return Ok(target); + } + // Existing links or directories occupy a name too. Windows may report + // AccessDenied rather than AlreadyExists when persisting over a directory. + match fs::symlink_metadata(&target) { + Ok(_) => continue, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect output filename: {error}")), + } + match file.file.persist_noclobber(&target) { + Ok(_) => { + crate::hot_folder::sync_parent(&target)?; + return Ok(target); + } + Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => file.file = error.file, + Err(error) => return Err(format!("cannot publish {}: {error}", target.display())), + } + } + Err("10000 output filenames already exist; no file was overwritten".into()) +} + +fn same_file(path: &Path, size: u64, hash: &str) -> Result { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(format!("cannot inspect output: {error}")), + }; + if !metadata.is_file() || metadata.file_type().is_symlink() || metadata.len() != size { + return Ok(false); + } + let mut reader = File::open(path).map_err(|error| error.to_string())?.take(size.saturating_add(1)); + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 32 * 1024]; + loop { + let count = reader.read(&mut buffer).map_err(|error| error.to_string())?; + if count == 0 { + break; + } + hasher.update(buffer.get(..count).ok_or("invalid output read size")?); + } + Ok(hasher.finalize().iter().map(|byte| format!("{byte:02x}")).collect::() == hash) +} + +fn safe_subfolder(base: &Path, relative: &Path) -> Result { + let mut path = base.to_owned(); + for part in relative.components() { + if !matches!(part, Component::Normal(_)) { + return Err("invalid output hierarchy".into()); + } + path.push(part); + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { + return Err("output hierarchy contains a link or a file".into()); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir(&path).map_err(|error| error.to_string())?; + crate::hot_folder::sync_parent(&path)?; + } + Err(error) => return Err(error.to_string()), + } + if !path.canonicalize().map_err(|error| error.to_string())?.starts_with(base) { + return Err("output hierarchy escaped its folder".into()); + } + } + path.canonicalize().map_err(|error| error.to_string()) +} + +const TOKENS: &[&str] = &["name", "folder", "task", "date", "time", "counter"]; +fn forbidden_name_char(c: char) -> bool { + c.is_control() || matches!(c, '<' | '>' | ':' | '"' | '/' | '\\' | '|' | '?' | '*') +} + +fn validate_template(template: &str) -> Result<(), String> { + if template.trim().is_empty() || template.len() > 200 || template.chars().any(forbidden_name_char) { + return Err("output filename must be 1..200 bytes without path separators or reserved filename characters".into()); + } + let mut remaining = template; + while let Some(open) = remaining.find('{') { + let before = remaining.get(..open).ok_or("invalid filename template")?; + if before.contains('}') { + return Err("unmatched '}' in output filename".into()); + } + let rest = remaining.get(open.saturating_add(1)..).ok_or("invalid filename template")?; + let end = rest.find('}').ok_or("unmatched '{' in output filename")?; + let token = rest.get(..end).ok_or("invalid filename token")?; + if !TOKENS.contains(&token) { + return Err(format!("unknown filename token {{{token}}}; use name, folder, task, date, time or counter")); + } + remaining = rest.get(end.saturating_add(1)..).ok_or("invalid filename template")?; + } + if remaining.contains('}') { + return Err("unmatched '}' in output filename".into()); + } + Ok(()) +} + +fn safe_component(value: &str) -> String { + let value: String = value.chars().map(|c| if forbidden_name_char(c) || matches!(c, '{' | '}') { '_' } else { c }).collect(); + let value = value.trim_matches([' ', '.']); + if value.is_empty() { + "output".into() + } else { + let mut component = String::new(); + for character in value.chars() { + if component.len().saturating_add(character.len_utf8()) > 180 { + break; + } + component.push(character); + } + component + } +} + +fn expand_name(template: &str, name: &str, folder: &str, task: &str, timestamp: DateTime, counter: usize) -> Result { + validate_template(template)?; + let mut name = template + .replace("{name}", &safe_component(name)) + .replace("{folder}", &safe_component(folder)) + .replace("{task}", &safe_component(task)) + .replace("{date}", ×tamp.format("%Y%m%d").to_string()) + .replace("{time}", ×tamp.format("%H%M%S").to_string()) + .replace("{counter}", &format!("{counter:04}")); + name = safe_component(&name); + let reserved = name.split('.').next().unwrap_or_default().to_ascii_uppercase(); + if matches!( + reserved.as_str(), + "CON" | "PRN" | "AUX" | "NUL" | "CLOCK$" | "CONIN$" | "CONOUT$" | "COM¹" | "COM²" | "COM³" | "LPT¹" | "LPT²" | "LPT³" + ) || (1..=9).any(|n| reserved == format!("COM{n}") || reserved == format!("LPT{n}")) + { + name.insert(0, '_'); + } + Ok(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn input(name: &str, text: &str) -> Input { + Input { relative: name.into(), bytes: Session::new().create_from_text(name, text).unwrap() } + } + fn run(inputs: &[Input], folder: &Path, steps: Vec) -> Result { + save(inputs, folder, &OutputPlan { steps }, "Batch", 1_700_000_000, None, &|| false) + } + fn text(path: &Path) -> String { + let mut session = Session::new(); + let id = session.open("result.pdf", None, Arc::new(fs::read(path).unwrap()), None).unwrap(); + let doc = session.get(id).unwrap(); + Exporter::new(doc).text_of(&(0..doc.info.pages.len()).collect::>()).unwrap() + } + + #[test] + fn groups_by_file_folder_and_all_and_preserves_page_order() { + let dir = tempfile::tempdir().unwrap(); + let inputs = [input("one/a.pdf", "Alpha first"), input("one/b.pdf", "Beta second"), input("two/c.pdf", "Gamma third")]; + let steps = vec![ + SaveStep::default(), + SaveStep { grouping: Grouping::Folder, name_template: "folder-{folder}".into(), ..Default::default() }, + SaveStep { grouping: Grouping::All, name_template: "all-{task}".into(), ..Default::default() }, + ]; + let receipt = run(&inputs, dir.path(), steps).unwrap(); + assert_eq!(receipt.outputs.len(), 6); + assert!(dir.path().join("one/a.pdf").is_file()); + let folder = text(&dir.path().join("one/folder-one.pdf")); + assert!(folder.find("Alpha first").unwrap() < folder.find("Beta second").unwrap()); + let all = text(&dir.path().join("all-Batch.pdf")); + assert!(all.find("Beta second").unwrap() < all.find("Gamma third").unwrap()); + assert!(!folder.contains("Gamma third")); + } + + #[test] + fn saves_multiple_real_export_formats_and_an_image_only_pdf() { + let dir = tempfile::tempdir().unwrap(); + let steps = Format::ALL + .iter() + .map(|format| SaveStep { + format: *format, + name_template: if *format == Format::ImagePdf { "image-only".into() } else { "result".into() }, + dpi: 72.0, + ..Default::default() + }) + .collect(); + let receipt = run(&[input("source.pdf", "Text survives editable export")], dir.path(), steps).unwrap(); + assert_eq!(receipt.outputs.len(), Format::ALL.len()); + assert!(fs::read_to_string(dir.path().join("result.txt")).unwrap().contains("Text survives")); + assert!(fs::read_to_string(dir.path().join("result.html")).unwrap().contains("Text survives")); + assert!(fs::read(dir.path().join("result.docx")).unwrap().starts_with(b"PK\x03\x04")); + for extension in ["xlsx", "epub"] { + assert!(fs::read(dir.path().join(format!("result.{extension}"))).unwrap().starts_with(b"PK\x03\x04")); + } + let analysis: serde_json::Value = serde_json::from_slice(&fs::read(dir.path().join("result.json")).unwrap()).unwrap(); + assert_eq!(analysis["format"], "pdfcraft-analysis"); + assert_eq!(analysis["pages"][0]["content"], "existing_text"); + assert!(fs::read_to_string(dir.path().join("result.rtf")).unwrap().starts_with("{\\rtf")); + for extension in ["png", "jpg", "tif"] { + let bytes = fs::read(dir.path().join(format!("result_page_0001.{extension}"))).unwrap(); + let image = image::load_from_memory(&bytes).unwrap(); + assert!(image.width() > 100 && image.height() > 100); + } + assert!(text(&dir.path().join("image-only.pdf")).trim().is_empty()); + assert!(text(&dir.path().join("result.pdf")).contains("Text survives")); + // Raster-only export retains document geometry instead of defaulting to pixel=point. + let mut session = Session::new(); + let id = session.open("image.pdf", None, Arc::new(fs::read(dir.path().join("image-only.pdf")).unwrap()), None).unwrap(); + let page = &session.get(id).unwrap().info.pages[0]; + assert!((page.width - 612.0).abs() < 1.0 && (page.height - 792.0).abs() < 1.0); + } + + #[test] + fn conversion_failure_never_publishes_the_successful_first_step() { + let dir = tempfile::tempdir().unwrap(); + let inputs = [input("valid.pdf", "Keep me"), Input { relative: "broken.pdf".into(), bytes: Arc::new(b"broken pdf".to_vec()) }]; + let error = run(&inputs, dir.path(), vec![SaveStep::default()]).unwrap_err(); + assert!(error.outputs.is_empty()); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 0); + assert_eq!(*inputs[1].bytes, b"broken pdf"); + } + + #[test] + fn occupied_names_are_never_overwritten_and_identical_retry_reuses_results() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("source.pdf"), b"unrelated user document").unwrap(); + let inputs = [input("source.pdf", "Keep recognised text")]; + let first = run(&inputs, dir.path(), vec![SaveStep::default()]).unwrap(); + assert!(first.outputs[0].ends_with("source (1).pdf")); + let second = run(&inputs, dir.path(), vec![SaveStep::default()]).unwrap(); + assert_eq!(first.outputs, second.outputs); + assert_eq!(fs::read(dir.path().join("source.pdf")).unwrap(), b"unrelated user document"); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 2); + fs::create_dir(dir.path().join("folder.pdf")).unwrap(); + let next = run(&[input("folder.pdf", "kept separate")], dir.path(), vec![SaveStep::default()]).unwrap(); + assert!(next.outputs[0].ends_with("folder (1).pdf")); + assert!(dir.path().join("folder.pdf").is_dir()); + } + + #[test] + fn template_and_path_boundaries_reject_escapes_and_use_batch_timestamp() { + let dir = tempfile::tempdir().unwrap(); + for template in ["../escape", "C:\\escape", "{unknown}", "bad{name", "bad}name", "a/b", "{name}:test"] { + assert!(run(&[input("good.pdf", "text")], dir.path(), vec![SaveStep { name_template: template.into(), ..Default::default() }]).is_err()); + } + for relative in ["../escape.pdf", "/outside.pdf", "child/../../escape.pdf"] { + assert!(run(&[input(relative, "text")], dir.path(), vec![SaveStep::default()]).is_err()); + } + let result = run( + &[input("x.pdf", "text")], + dir.path(), + vec![SaveStep { name_template: "{task}-{date}-{time}-{counter}-{name}".into(), ..Default::default() }], + ) + .unwrap(); + assert!(result.outputs[0].ends_with("Batch-20231114-221320-0001-x.pdf")); + let result = run(&[input("CON.pdf", "text")], dir.path(), vec![SaveStep::default()]).unwrap(); + assert!(result.outputs[0].ends_with("_CON.pdf")); + assert!(run(&[input("x.pdf", "text")], dir.path(), vec![SaveStep { dpi: f64::NAN, ..Default::default() }]).is_err()); + } + + #[test] + fn stopped_publishing_reports_complete_outputs_and_keeps_the_inputs() { + let dir = tempfile::tempdir().unwrap(); + let inputs = [input("a.pdf", "first"), input("b.pdf", "second")]; + let error = save(&inputs, dir.path(), &OutputPlan::default(), "Batch", 0, None, &|| dir.path().join("a.pdf").exists()).unwrap_err(); + assert_eq!(error.outputs.len(), 1); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1); + assert!(text(&error.outputs[0]).contains("first")); + assert!(inputs.iter().all(|input| !input.bytes.is_empty())); + } + + #[test] + fn root_and_subfolder_file_collisions_are_checked_before_publication() { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let plan = OutputPlan { steps: vec![SaveStep { folder: Some(outside.path().to_owned()), ..Default::default() }] }; + assert!(save(&[input("a.pdf", "text")], root.path(), &plan, "Batch", 0, Some(root.path()), &|| false).is_err()); + fs::write(root.path().join("child"), b"must remain").unwrap(); + assert!(run(&[input("child/a.pdf", "text")], root.path(), vec![SaveStep::default()]).is_err()); + assert_eq!(fs::read(root.path().join("child")).unwrap(), b"must remain"); + } + + #[test] + fn absent_plan_fields_keep_the_original_one_pdf_per_file_behavior() { + let plan: OutputPlan = serde_json::from_str("{}").unwrap(); + assert_eq!(plan.steps.len(), 1); + assert!(!plan.needs_batch()); + assert_eq!(plan.steps[0].format, Format::Pdf); + assert_eq!(plan.steps[0].name_template, "{name}"); + for unsupported in ["djvu", "fb2", "csv"] { + assert!(serde_json::from_str::(&format!("\"{unsupported}\"")).is_err()); + } + } +} diff --git a/crates/engine/src/hot_folder_preprocess.rs b/crates/engine/src/hot_folder_preprocess.rs new file mode 100644 index 0000000000..cc2fabc17a --- /dev/null +++ b/crates/engine/src/hot_folder_preprocess.rs @@ -0,0 +1,299 @@ +//! Reusable Hot Folder page selection, image preparation and recognition-area templates. +//! Image changes are opt-in and preserve physical size at the actual rendering resolution. +use crate::{Session, ocr}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::io::Cursor; +use std::sync::Arc; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Settings { + /// One-based page ranges, e.g. 1-3,5. Empty selects the whole document. + pub pages: String, + pub grayscale: bool, + pub invert: bool, + pub brightness: i32, + pub contrast: f32, + pub threshold: Option, + pub median_filter: bool, + /// Split a spread vertically into a left and right page, in reading order. + pub split_spreads: bool, + pub regions: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Region { + /// Zero applies to every prepared page; otherwise a one-based prepared page number. + pub page: usize, + /// Fractions of displayed page size, from top-left: x, y, width, height. + pub rect: [f32; 4], + /// Included areas constrain recognition; excluded areas suppress words inside them. + pub exclude: bool, +} +impl Default for Region { + fn default() -> Self { + Self { page: 0, rect: [0.0, 0.0, 1.0, 1.0], exclude: false } + } +} + +impl Settings { + pub fn changes_images(&self) -> bool { + self.grayscale + || self.invert + || self.brightness != 0 + || self.contrast != 0.0 + || self.threshold.is_some() + || self.median_filter + || self.split_spreads + } + pub fn validate(&self) -> Result<(), String> { + if self.pages.len() > 4096 + || self.regions.len() > 64 + || !(-100..=100).contains(&self.brightness) + || !self.contrast.is_finite() + || !(-100.0..=100.0).contains(&self.contrast) + { + return Err("image settings exceed allowed ranges (brightness/contrast -100..100, at most 64 regions)".into()); + } + parse_pages(&self.pages, None)?; + for region in &self.regions { + let [x, y, w, h] = region.rect; + if !region.rect.iter().all(|v| v.is_finite()) + || x < 0.0 + || y < 0.0 + || w <= 0.0 + || h <= 0.0 + || x + w > 1.00001 + || y + h > 1.00001 + || region.page > 10_000 + { + return Err("recognition regions must fit within the page (coordinates 0..1)".into()); + } + } + Ok(()) + } +} + +pub fn parse_pages(value: &str, count: Option) -> Result, String> { + if value.trim().is_empty() { + return Ok(count.map_or_else(Vec::new, |n| (0..n).collect())); + } + let mut pages = BTreeSet::new(); + for range in value.split(',') { + let mut parts = range.trim().split('-'); + let first = + parts.next().unwrap_or_default().trim().parse::().map_err(|_| "pages must be one-based numbers/ranges, for example 1-3,5")?; + let last = parts.next().map(|v| v.trim().parse::()).transpose().map_err(|_| "invalid page range")?.unwrap_or(first); + if parts.next().is_some() || first == 0 || last < first || last > count.unwrap_or(10_000) { + return Err("page range is reversed or outside the document".into()); + } + for page in first..=last { + pages.insert(page - 1); + } + } + Ok(pages.into_iter().collect()) +} + +pub fn prepare(bytes: Arc>, settings: &Settings, dpi: f64, should_stop: &dyn Fn() -> bool) -> Result>, String> { + settings.validate()?; + if !settings.changes_images() && settings.pages.trim().is_empty() { + return Ok(bytes); + } + let mut session = Session::new(); + let id = session.open("input.pdf", None, bytes, None).map_err(|e| e.to_string())?; + let doc = session.get(id).ok_or("document unavailable")?; + if let Some(reason) = &doc.read_only_reason { + return Err(reason.clone()); + } + let selected = parse_pages(&settings.pages, Some(doc.info.pages.len()))?; + if !settings.changes_images() { + return session.extract(id, &selected).map_err(|e| e.to_string()); + } + // A raster replacement cannot preserve interactive structures. Refuse to silently flatten them. + if !doc.info.annotations.is_empty() || !doc.info.fields.is_empty() || !doc.info.links.is_empty() { + return Err("image preprocessing cannot preserve annotations, links or forms; disable image changes for this document".into()); + } + let mut exporter = crate::export::Exporter::new(doc); + let mut sources = Vec::new(); + let mut total = 0usize; + for page in selected { + if should_stop() { + return Err("task stopped during image preprocessing".into()); + } + let png = exporter.png(page, dpi)?; + let mut rgb = image::load_from_memory(&png).map_err(|e| e.to_string())?.to_rgb8(); + transform(&mut rgb, settings); + let slices = if settings.split_spreads { + vec![(0, rgb.width() / 2), (rgb.width() / 2, rgb.width() - rgb.width() / 2)] + } else { + vec![(0, rgb.width())] + }; + for (left, width) in slices { + if width == 0 { + return Err("page is too narrow to split".into()); + } + let image = image::imageops::crop_imm(&rgb, left, 0, width, rgb.height()).to_image(); + let mut encoded = Cursor::new(Vec::new()); + image::DynamicImage::ImageRgb8(image).write_to(&mut encoded, image::ImageFormat::Png).map_err(|e| e.to_string())?; + total = total.saturating_add(encoded.get_ref().len()); + if total > 512 * 1024 * 1024 { + return Err("prepared images exceed 512 MiB; reduce resolution or select fewer pages".into()); + } + let name = format!("page-{}-{left}.png", page.saturating_add(1)); + let pdf = session + .create_from_images_with_resolution( + &[(name.clone(), encoded.into_inner())], + crate::ImageResolution::Dpi(exporter.dpi_used(page, dpi)), + ) + .map_err(|e| e.to_string())?; + sources.push((name, pdf)); + } + } + session.combine(&sources).map_err(|e| e.to_string()) +} + +fn transform(image: &mut image::RgbImage, settings: &Settings) { + if settings.median_filter { + let source = image.clone(); + for (x, y, pixel) in image.enumerate_pixels_mut() { + for (channel, value) in pixel.0.iter_mut().enumerate() { + let mut values = [255u8; 9]; + let mut count = 0usize; + for row in y.saturating_sub(1)..=y.saturating_add(1).min(source.height().saturating_sub(1)) { + for col in x.saturating_sub(1)..=x.saturating_add(1).min(source.width().saturating_sub(1)) { + if let Some(value) = values.get_mut(count) { + *value = source.get_pixel(col, row).0.get(channel).copied().unwrap_or(255); + count += 1; + } + } + } + if let Some(values) = values.get_mut(..count) { + values.sort_unstable(); + *value = values.get(count / 2).copied().unwrap_or(*value); + } + } + } + } + let factor = ((100.0 + settings.contrast) / 100.0).powi(2); + for pixel in image.pixels_mut() { + for value in &mut pixel.0 { + let level = (f32::from(*value) - 127.5) * factor + 127.5 + settings.brightness as f32 * 2.55; + *value = level.round().clamp(0.0, 255.0) as u8; + if settings.invert { + *value = 255 - *value; + } + } + if settings.grayscale || settings.threshold.is_some() { + let [r, g, b] = pixel.0; + let mut level = ((u32::from(r) * 299 + u32::from(g) * 587 + u32::from(b) * 114) / 1000) as u8; + if let Some(threshold) = settings.threshold { + level = if level < threshold { 0 } else { 255 }; + } + pixel.0 = [level; 3]; + } + } +} + +/// Regions constrain only new OCR text; visible page content is preserved. +pub fn constrain_words(pages: &mut [ocr::OcrPage], infos: &[pdfcraft_render::PageInfo], regions: &[Region]) { + for page in pages { + let Some(info) = infos.get(page.page) else { continue }; + let applies = |r: &&Region| r.page == 0 || r.page == page.page.saturating_add(1); + let selected: Vec<_> = regions.iter().filter(applies).collect(); + page.words.retain(|word| { + let x = word.origin[0] + (word.across[0] + word.up[0]) * 0.5; + let y = word.origin[1] + (word.across[1] + word.up[1]) * 0.5; + let [x, y] = info.user_to_view(x as f32, y as f32); + let x = x / info.width.max(0.001); + let y = y / info.height.max(0.001); + let contains = |r: &&Region| { + let [left, top, w, h] = r.rect; + x >= left && x <= left + w && y >= top && y <= top + h + }; + (!selected.iter().any(|r| !r.exclude) || selected.iter().filter(|r| !r.exclude).any(contains)) + && !selected.iter().filter(|r| r.exclude).any(contains) + }); + } +} + +pub struct BlankResult { + pub document: Option>>, + pub blank_pages: Option>>, +} +/// Strict white-only detection retains faint marks, existing text and interactive content. +pub fn separate_blank_pages(bytes: Arc>, should_stop: &dyn Fn() -> bool) -> Result { + let mut session = Session::new(); + let id = session.open("scan.pdf", None, bytes.clone(), None).map_err(|e| e.to_string())?; + let doc = session.get(id).ok_or("document unavailable")?; + if !doc.info.annotations.is_empty() || !doc.info.fields.is_empty() || !doc.info.links.is_empty() { + return Ok(BlankResult { document: Some(bytes), blank_pages: None }); + } + let count = doc.info.pages.len(); + let mut exporter = crate::export::Exporter::new(doc); + let mut blank = Vec::new(); + for page in 0..count { + if should_stop() { + return Err("task stopped while detecting blank pages".into()); + } + if !exporter.text(page)?.trim().is_empty() { + continue; + } + let png = exporter.png(page, 150.0)?; + let image = image::load_from_memory(&png).map_err(|e| e.to_string())?.to_rgb8(); + if image.pixels().all(|p| p.0 == [255; 3]) { + blank.push(page); + } + } + if blank.is_empty() { + return Ok(BlankResult { document: Some(bytes), blank_pages: None }); + } + let archived = session.extract(id, &blank).map_err(|e| e.to_string())?; + let keep: Vec<_> = (0..count).filter(|p| !blank.contains(p)).collect(); + let document = if keep.is_empty() { None } else { Some(session.extract(id, &keep).map_err(|e| e.to_string())?) }; + Ok(BlankResult { document, blank_pages: Some(archived) }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn hot_folder_page_ranges_are_bounded_and_sorted() { + assert_eq!(parse_pages("3,1-2,2", Some(3)).unwrap(), vec![0, 1, 2]); + for value in ["0", "3-1", "1-10001", "NaN", "1-2-3"] { + assert!(parse_pages(value, Some(10)).is_err()); + } + } + #[test] + fn hot_folder_image_options_change_pixels_without_touching_the_source() { + let mut image = image::RgbImage::from_pixel(3, 3, image::Rgb([0, 0, 0])); + image.put_pixel(1, 1, image::Rgb([255, 255, 255])); + transform(&mut image, &Settings { median_filter: true, invert: true, ..Default::default() }); + assert!(image.pixels().all(|p| p.0 == [255; 3])); + let mut image = image::RgbImage::from_pixel(2, 2, image::Rgb([200, 0, 0])); + transform(&mut image, &Settings { threshold: Some(128), ..Default::default() }); + assert!(image.pixels().all(|p| p.0 == [0; 3])); + } + #[test] + fn hot_folder_blank_pages_can_be_archived_even_when_all_are_blank() { + let bytes = Session::new().create_blank(100.0, 100.0, 2).unwrap(); + let result = separate_blank_pages(bytes, &|| false).unwrap(); + assert!(result.document.is_none()); + let mut s = Session::new(); + let id = s.open("blank", None, result.blank_pages.unwrap(), None).unwrap(); + assert_eq!(s.get(id).unwrap().info.pages.len(), 2); + } + #[test] + fn hot_folder_spread_split_preserves_physical_size() { + let bytes = Session::new().create_blank(200.0, 100.0, 1).unwrap(); + let result = prepare(bytes, &Settings { split_spreads: true, ..Default::default() }, 72.0, &|| false).unwrap(); + let mut s = Session::new(); + let id = s.open("split", None, result, None).unwrap(); + let pages = &s.get(id).unwrap().info.pages; + assert_eq!(pages.len(), 2); + for page in pages { + assert!((page.width - 100.0).abs() < 0.1 && (page.height - 100.0).abs() < 0.1); + } + } +} diff --git a/crates/engine/src/hot_folder_schedule.rs b/crates/engine/src/hot_folder_schedule.rs new file mode 100644 index 0000000000..fc8a530d94 --- /dev/null +++ b/crates/engine/src/hot_folder_schedule.rs @@ -0,0 +1,97 @@ +//! Calendar schedules in the machine's local timezone. Nonexistent DST times move to the +//! next valid local minute; repeated times run once at their earlier occurrence. +use chrono::{Datelike, Local, TimeZone, Timelike}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Schedule { + #[default] + Interval, + Daily { + hour: u32, + minute: u32, + }, + Weekly { + weekday: u32, + hour: u32, + minute: u32, + }, + Monthly { + day: u32, + hour: u32, + minute: u32, + }, +} + +impl Schedule { + pub fn validate(self) -> Result<(), String> { + let (hour, minute) = match self { + Self::Interval => return Ok(()), + Self::Daily { hour, minute } => (hour, minute), + Self::Weekly { weekday, hour, minute } if weekday < 7 => (hour, minute), + Self::Monthly { day, hour, minute } if (1..=31).contains(&day) => (hour, minute), + _ => return Err("weekday must be 0..6 (Monday..Sunday); month day must be 1..31".into()), + }; + if hour > 23 || minute > 59 { + return Err("schedule time must be 00:00..23:59".into()); + } + Ok(()) + } + + pub fn next_after(self, after: u64) -> Result { + self.next_in(after, Local) + } + + fn next_in(self, after: u64, zone: T) -> Result { + self.validate()?; + let origin = i64::try_from(after).ok().and_then(|v| zone.timestamp_opt(v, 0).single()).ok_or("invalid schedule timestamp")?; + let mut date = origin.date_naive(); + for _ in 0..370 { + let (matches, hour, minute) = match self { + Self::Interval => return Ok(after.saturating_add(1)), + Self::Daily { hour, minute } => (true, hour, minute), + Self::Weekly { weekday, hour, minute } => (date.weekday().num_days_from_monday() == weekday, hour, minute), + Self::Monthly { day, hour, minute } => { + let last = date.succ_opt().is_some_and(|d| d.month() != date.month()); + (date.day() == day || last && date.day() < day, hour, minute) + } + }; + if matches { + let mut local = date.and_hms_opt(hour, minute, 0).ok_or("invalid schedule time")?; + // A DST gap is bounded to this date; do not drift a daily task into tomorrow. + while local.date() == date { + if let Some(candidate) = zone.from_local_datetime(&local).earliest() { + if candidate.timestamp() > origin.timestamp() { + return u64::try_from(candidate.timestamp()).map_err(|e| e.to_string()); + } + break; + } + local = local.checked_add_signed(chrono::Duration::minutes(1)).ok_or("schedule overflow")?; + if local.hour() == 0 && local.minute() == 0 { + break; + } + } + } + date = date.succ_opt().ok_or("schedule date overflow")?; + } + Err("no valid schedule occurrence within the next year".into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn calendar_schedules_preserve_time_and_clamp_short_months() { + let tz = chrono::Utc; + let stamp = |year, month, day, hour, minute| { + u64::try_from(tz.with_ymd_and_hms(year, month, day, hour, minute, 0).single().unwrap().timestamp()).unwrap() + }; + let start = stamp(2026, 1, 31, 8, 0); + assert_eq!(Schedule::Daily { hour: 8, minute: 0 }.next_in(start, tz).unwrap(), stamp(2026, 2, 1, 8, 0)); + assert_eq!(Schedule::Monthly { day: 31, hour: 8, minute: 0 }.next_in(start, tz).unwrap(), stamp(2026, 2, 28, 8, 0)); + assert_eq!(Schedule::Weekly { weekday: 0, hour: 9, minute: 30 }.next_in(start, tz).unwrap(), stamp(2026, 2, 2, 9, 30)); + assert!(Schedule::Weekly { weekday: 7, hour: 9, minute: 0 }.validate().is_err()); + } +} diff --git a/crates/engine/src/hot_folder_sources.rs b/crates/engine/src/hot_folder_sources.rs new file mode 100644 index 0000000000..e0dadae6fc --- /dev/null +++ b/crates/engine/src/hot_folder_sources.rs @@ -0,0 +1,786 @@ +//! Explicit Hot Folder source adapters. No network access occurs until a worker calls +//! `stage` or a user starts sign-in. Task configuration and journals contain no secrets. +//! Remote originals are acknowledged only after the caller has durably saved every output. + +mod credentials; +mod ftp; +mod outlook; +mod project; + +pub use credentials::{CredentialStore, credential_status, forget_credentials, save_ftp_password}; +pub use outlook::{DeviceSignIn, MailFolder, SignInStatus, outlook_begin_sign_in, outlook_folders, outlook_poll_sign_in}; +pub use project::{ProjectManifest, write_project_manifest}; + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +const MAX_FILE: u64 = 512 * 1024 * 1024; +const MAX_ITEMS: usize = 10_000; +const MAX_BATCH: usize = 128; +const MAX_JSON: u64 = 8 * 1024 * 1024; +const STATE_FILE: &str = ".pdfcraft-source-state.json"; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SourceConfig { + #[default] + Local, + Ftp(FtpSource), + Outlook(OutlookSource), + Project(ProjectSource), +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SourceAction { + #[default] + Keep, + /// FTP: existing absolute server directory. Outlook: destination mail-folder ID. + /// Project: existing local directory. Outlook actions affect the entire message. + Move { + destination: String, + }, + Delete, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FtpSecurity { + #[default] + ExplicitTls, + /// Clear-text FTP is available only when explicitly selected; no TLS downgrade. + Plain, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct FtpSource { + pub host: String, + pub port: u16, + pub security: FtpSecurity, + pub username: String, + pub credential_id: String, + pub folder: String, + pub recursive: bool, + pub action: SourceAction, +} +impl Default for FtpSource { + fn default() -> Self { + Self { + host: String::new(), + port: 21, + security: FtpSecurity::ExplicitTls, + username: String::new(), + credential_id: String::new(), + folder: "/".into(), + recursive: false, + action: SourceAction::Keep, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct OutlookSource { + /// The user's own registered public-client application. There is no bundled client ID. + pub client_id: String, + pub tenant: String, + pub credential_id: String, + pub folder_id: String, + pub action: SourceAction, +} +impl Default for OutlookSource { + fn default() -> Self { + Self { + client_id: String::new(), + tenant: "common".into(), + credential_id: String::new(), + folder_id: "inbox".into(), + action: SourceAction::Keep, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ProjectSource { + pub manifest: PathBuf, + pub action: SourceAction, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StagedSource { + pub source_id: String, + /// Original logical path, suitable for output grouping; never used for staging writes. + pub relative: PathBuf, + /// A single safe filename beneath the task's staging inbox. + pub staged_relative: PathBuf, + pub sha256: String, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct StageReport { + pub items: Vec, + pub warnings: Vec, + #[serde(default)] + pub has_more: bool, + /// Output already confirmed; caller must recheck its durable output receipt before retrying ack. + #[serde(default)] + pub pending_acknowledgements: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct RemoteFile { + key: String, + relative: PathBuf, + revision: String, + size: u64, + /// For Outlook: immutable message ID. An action waits for every supported attachment. + group: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Record { + source: StagedSource, + remote: RemoteFile, + ready: bool, + acknowledged: bool, +} + +#[derive(Default, Serialize, Deserialize)] +struct Journal { + config_hash: String, + records: BTreeMap, +} + +pub fn validate(config: &SourceConfig, input: &Path, root: Option<&Path>) -> Result<(), String> { + let input = confined(input, root)?; + if !input.is_dir() { + return Err("source inbox must be an existing directory".into()); + } + match config { + SourceConfig::Local => Ok(()), + SourceConfig::Ftp(source) => ftp::validate(source), + SourceConfig::Outlook(source) => outlook::validate(source), + SourceConfig::Project(source) => project::validate(source, &input, root), + } +} + +/// Local originals referenced by a source, for cross-task source/output overlap checks. +/// Completed project entries retain their confined path even after a successful move/delete. +pub fn local_paths(config: &SourceConfig, root: Option<&Path>) -> Result, String> { + guarded(|| match config { + SourceConfig::Project(source) => project::local_paths(source, root), + _ => Ok(Vec::new()), + }) +} + +trait Backend { + fn list(&mut self, stop: &Control<'_>) -> Result, String>; + fn read(&mut self, file: &RemoteFile, writer: &mut dyn Write, stop: &Control<'_>) -> Result<(), String>; + fn finish(&mut self, files: &[Record], action: &SourceAction, stop: &Control<'_>) -> Result<(), String>; +} + +fn backend<'a>( + config: &'a SourceConfig, + root: Option<&'a Path>, + control: &Control<'_>, + store: &'a CredentialStore, +) -> Result, String> { + match config { + SourceConfig::Ftp(source) => Ok(Box::new(ftp::Client::connect(store, source, control)?)), + SourceConfig::Outlook(source) => Ok(Box::new(outlook::Client::connect(store, source)?)), + SourceConfig::Project(source) => Ok(Box::new(project::Client::new(source, root)?)), + SourceConfig::Local => Err("local folders do not use a source adapter".into()), + } +} + +fn action(config: &SourceConfig) -> &SourceAction { + match config { + SourceConfig::Ftp(source) => &source.action, + SourceConfig::Outlook(source) => &source.action, + SourceConfig::Project(source) => &source.action, + SourceConfig::Local => &SourceAction::Keep, + } +} + +/// Blocking worker call. Returns both new and previously staged, unacknowledged items. +/// Stage only into this task's inbox; secrets are resolved from the explicitly selected store. +pub fn stage( + config: &SourceConfig, + input: &Path, + root: Option<&Path>, + should_stop: &dyn Fn() -> bool, + store: &CredentialStore, +) -> Result { + guarded(|| { + validate(config, input, root)?; + if matches!(config, SourceConfig::Local) { + return Ok(StageReport::default()); + } + let input = confined(input, root)?; + let _lock = lock(&input)?; + let mut journal = read_journal(&input, config)?; + let control = Control::new(should_stop); + control.check()?; + let mut source = backend(config, root, &control, store)?; + let mut report = stage_with(source.as_mut(), config, &input, &mut journal, &control)?; + if let SourceConfig::Project(project) = config { + project::validate_and_order(project, root, &journal, &mut report)?; + } + Ok(report) + }) +} + +fn stage_with( + source: &mut dyn Backend, + config: &SourceConfig, + input: &Path, + journal: &mut Journal, + control: &Control<'_>, +) -> Result { + let files = source.list(control)?; + if files.len() > MAX_ITEMS { + return Err("source contains more than 10000 entries; choose a smaller source folder".into()); + } + let mut unique = BTreeSet::new(); + for file in &files { + if !unique.insert((&file.group, &file.key)) { + return Err("source returned duplicate file identifiers; originals retained".into()); + } + } + let mut report = StageReport::default(); + let mut total = 0_u64; + for remote in files { + control.check()?; + if !relative_ok(&remote.relative) || remote.size == 0 || remote.size > MAX_FILE { + return Err("source path is unsafe or source exceeds the 512 MiB per-file limit".into()); + } + let source_id = hash(format!("{}\n{}\n{}\n{}", journal.config_hash, remote.key, remote.revision, remote.size).as_bytes()); + if journal.records.get(&source_id).is_some_and(|record| record.acknowledged || record.ready) { + continue; + } + if report.items.len() >= MAX_BATCH || total.saturating_add(remote.size) > MAX_FILE { + report.warnings.push("source queue continues in the next batch (128 files / 512 MiB)".into()); + report.has_more = true; + break; + } + if let Some(record) = journal.records.get(&source_id) { + let path = input.join(&record.source.staged_relative); + if file_hash(&path, control).is_ok_and(|value| value == record.source.sha256) { + total = total.saturating_add(remote.size); + report.items.push(record.source.clone()); + continue; + } + if path.exists() { + return Err("staged source changed locally; retain it and use a fresh staging inbox".into()); + } + } + if journal.records.len() >= MAX_ITEMS { + return Err("source journal reached 10000 versions; archive the task and create a new task".into()); + } + let name = remote.relative.file_name().and_then(|name| name.to_str()).ok_or("source filename is not valid UTF-8")?; + let staged_relative = PathBuf::from(format!("{}-{}", source_id.get(..24).ok_or("invalid source ID")?, safe_filename(name))); + let mut temporary = tempfile::Builder::new().prefix(".pdfcraft-source-").tempfile_in(input).map_err(io_error)?; + source.read(&remote, &mut temporary, control)?; + let size = temporary.as_file().metadata().map_err(io_error)?.len(); + if size != remote.size { + return Err("source changed during download (size differs); original retained".into()); + } + temporary.as_file().sync_all().map_err(io_error)?; + let sha256 = file_hash(temporary.path(), control)?; + let target = input.join(&staged_relative); + match temporary.persist_noclobber(&target) { + Ok(_) => {} + Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { + if file_hash(&target, control)? != sha256 { + return Err("staging filename is occupied by different content; no file was replaced".into()); + } + } + Err(error) => return Err(io_error(error.error)), + } + let item = StagedSource { source_id: source_id.clone(), relative: remote.relative.clone(), staged_relative, sha256 }; + journal.records.insert(source_id, Record { source: item.clone(), remote, ready: false, acknowledged: false }); + write_journal(input, journal)?; + total = total.saturating_add(size); + report.items.push(item); + } + // Previously staged inputs remain actionable even when their original was moved by an + // external actor. A destructive acknowledgement must still verify its remote version. + for record in journal.records.values().filter(|record| !record.acknowledged && !record.ready) { + if report.items.iter().any(|item| item.source_id == record.source.source_id) { + continue; + } + if report.items.len() >= MAX_BATCH || total.saturating_add(record.remote.size) > MAX_FILE { + report.has_more = true; + break; + } + if !report.items.iter().any(|item| item.source_id == record.source.source_id) + && file_hash(&input.join(&record.source.staged_relative), control).is_ok_and(|value| value == record.source.sha256) + { + report.items.push(record.source.clone()); + total = total.saturating_add(record.remote.size); + } + } + report.pending_acknowledgements = + journal.records.values().filter(|record| record.ready && !record.acknowledged).map(|record| record.source.clone()).collect(); + let _ = config; + Ok(report) +} + +/// Call only after every output has been durably journaled. `confirmed_sha256` must be +/// the hash of the exact staged input used for those outputs. Idempotent on restart. +/// Does not remove staging files: the caller may do so only after this returns Ok. +/// For Outlook, Ok can mean this attachment is ready while another attachment still +/// needs processing; the remote message remains until the whole group is ready. +#[allow(clippy::too_many_arguments)] +pub fn acknowledge( + config: &SourceConfig, + input: &Path, + item: &StagedSource, + confirmed_sha256: &str, + root: Option<&Path>, + should_stop: &dyn Fn() -> bool, + store: &CredentialStore, +) -> Result<(), String> { + guarded(|| { + validate(config, input, root)?; + let input = confined(input, root)?; + let _lock = lock(&input)?; + let mut journal = read_journal(&input, config)?; + let control = Control::new(should_stop); + control.check()?; + let record = journal.records.get_mut(&item.source_id).ok_or("source receipt is absent from the durable journal")?; + if record.source != *item || confirmed_sha256 != record.source.sha256 { + return Err("output receipt does not match this source version; original retained".into()); + } + if record.acknowledged { + return Ok(()); + } + if !record.ready && file_hash(&input.join(&record.source.staged_relative), &control)? != confirmed_sha256 { + return Err("staged input changed after processing; original retained".into()); + } + record.ready = true; + let group = record.remote.group.clone(); + write_journal(&input, &journal)?; + let records = journal + .records + .values() + .filter(|record| { + !record.acknowledged + && if let Some(group) = &group { record.remote.group.as_ref() == Some(group) } else { record.source.source_id == item.source_id } + }) + .cloned() + .collect::>(); + if records.iter().any(|record| !record.ready) { + return Ok(()); + } + if !matches!(action(config), SourceAction::Keep) { + backend(config, root, &control, store)?.finish(&records, action(config), &control)?; + } + for record in records { + if let Some(record) = journal.records.get_mut(&record.source.source_id) { + record.acknowledged = true; + } + } + write_journal(&input, &journal) + }) +} + +fn guarded(run: impl FnOnce() -> Result) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(run)) + .unwrap_or_else(|_| Err("source adapter failed unexpectedly; originals retained".into())) +} + +struct Control<'a> { + stop: &'a dyn Fn() -> bool, + deadline: Instant, +} +impl<'a> Control<'a> { + fn new(stop: &'a dyn Fn() -> bool) -> Self { + Self { stop, deadline: Instant::now() + Duration::from_secs(300) } + } + fn check(&self) -> Result<(), String> { + if (self.stop)() { + Err("source operation cancelled; originals retained".into()) + } else if Instant::now() >= self.deadline { + Err("source operation exceeded five minutes; retry a smaller source folder".into()) + } else { + Ok(()) + } + } +} +fn io_error(error: impl std::fmt::Display) -> String { + format!("source I/O failed: {error}") +} +fn hash(bytes: &[u8]) -> String { + Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect() +} +fn relative_ok(path: &Path) -> bool { + !path.as_os_str().is_empty() && path.components().all(|part| matches!(part, Component::Normal(_))) +} +/// Reject symlink/junction components before canonicalization, even when their target +/// happens to be inside --root. Missing final components remain allowed for creation. +fn confined(path: &Path, root: Option<&Path>) -> Result { + let absolute = std::path::absolute(path).map_err(io_error)?; + for ancestor in absolute.ancestors() { + match fs::symlink_metadata(ancestor) { + Ok(metadata) if is_link(&metadata) => return Err("source paths must not contain symbolic links or junctions".into()), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(io_error(error)), + } + } + crate::hot_folder::confined(path, root) +} +fn is_link(metadata: &fs::Metadata) -> bool { + #[cfg(windows)] + { + use std::os::windows::fs::MetadataExt; + // All reparse points are refused, including directory junctions. + metadata.file_attributes() & 0x400 != 0 + } + #[cfg(not(windows))] + { + metadata.file_type().is_symlink() + } +} +fn supported(path: &Path) -> bool { + matches!( + path.extension().and_then(|v| v.to_str()).unwrap_or_default().to_ascii_lowercase().as_str(), + "pdf" | "png" | "jpg" | "jpeg" | "tif" | "tiff" + ) +} +fn safe_filename(value: &str) -> String { + let path = Path::new(value); + let extension = path.extension().and_then(|value| value.to_str()).filter(|_| supported(path)); + let stem = if extension.is_some() { path.file_stem().and_then(|value| value.to_str()).unwrap_or(value) } else { value }; + let mut result = String::new(); + for character in stem.chars() { + if result.len().saturating_add(character.len_utf8()) > 148 { + break; + } + result.push(if character.is_control() || matches!(character, '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|') { + '_' + } else { + character + }); + } + let mut result = result.trim_matches([' ', '.']).to_owned(); + if result.is_empty() { + result.push_str("source"); + } + if let Some(extension) = extension { + result.push('.'); + result.push_str(&extension.to_ascii_lowercase()); + } + result +} +fn file_hash(path: &Path, control: &Control<'_>) -> Result { + let metadata = fs::symlink_metadata(path).map_err(io_error)?; + if !metadata.is_file() || is_link(&metadata) || metadata.len() > MAX_FILE { + return Err("source must be a regular non-link file of at most 512 MiB".into()); + } + let mut file = File::open(path).map_err(io_error)?; + let mut digest = Sha256::new(); + let mut bytes = [0_u8; 64 * 1024]; + let mut size = 0_u64; + loop { + control.check()?; + let n = file.read(&mut bytes).map_err(io_error)?; + if n == 0 { + break; + } + size = size.saturating_add(n as u64); + if size > MAX_FILE { + return Err("source grew beyond 512 MiB".into()); + } + digest.update(bytes.get(..n).ok_or("invalid read size")?); + } + Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect()) +} +fn copy_limited(reader: &mut dyn Read, writer: &mut dyn Write, limit: u64, control: &Control<'_>) -> Result { + let mut total = 0_u64; + let mut buffer = [0_u8; 64 * 1024]; + loop { + control.check()?; + let n = reader.read(&mut buffer).map_err(|_| "source transfer failed or timed out; partial download was discarded".to_owned())?; + if n == 0 { + return Ok(total); + } + total = total.saturating_add(n as u64); + if total > limit { + return Err("source download exceeds its permitted size; partial download was discarded".into()); + } + writer.write_all(buffer.get(..n).ok_or("invalid transfer size")?).map_err(io_error)?; + } +} +fn plain_file(path: &Path) -> Result<(), String> { + match fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() && !is_link(&meta) => Ok(()), + Ok(_) => Err("source state must be a regular file, never a link".into()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(io_error(error)), + } +} +fn lock(input: &Path) -> Result { + let path = input.join(".pdfcraft-source.lock"); + plain_file(&path)?; + let file = OpenOptions::new().create(true).truncate(false).read(true).write(true).open(path).map_err(io_error)?; + file.try_lock().map_err(|_| "another worker is processing this source inbox".to_owned())?; + Ok(file) +} +fn read_journal(input: &Path, config: &SourceConfig) -> Result { + let path = input.join(STATE_FILE); + plain_file(&path)?; + let config_hash = hash(&serde_json::to_vec(config).map_err(io_error)?); + if !path.exists() { + return Ok(Journal { config_hash, ..Default::default() }); + } + let bytes = read_bounded(&path, MAX_JSON)?; + let journal: Journal = serde_json::from_slice(&bytes).map_err(|_| "source journal is malformed; originals retained".to_owned())?; + if journal.config_hash != config_hash { + return Err("source settings changed; choose a fresh staging inbox to avoid applying receipts to another source".into()); + } + if journal.records.len() > MAX_ITEMS { + return Err("source journal exceeds 10000 records".into()); + } + for (id, record) in &journal.records { + if id.len() != 64 + || !id.bytes().all(|byte| byte.is_ascii_hexdigit()) + || id != &record.source.source_id + || id != &hash(format!("{}\n{}\n{}\n{}", journal.config_hash, record.remote.key, record.remote.revision, record.remote.size).as_bytes()) + || !relative_ok(&record.source.staged_relative) + || record.source.staged_relative.components().count() != 1 + || record.source.sha256.len() != 64 + || !record.source.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) + || !relative_ok(&record.source.relative) + || record.source.relative != record.remote.relative + || record.remote.size == 0 + || record.remote.size > MAX_FILE + { + return Err("source journal contains an invalid receipt path or hash".into()); + } + } + Ok(journal) +} +fn write_journal(input: &Path, journal: &Journal) -> Result<(), String> { + let target = input.join(STATE_FILE); + plain_file(&target)?; + let bytes = serde_json::to_vec(journal).map_err(io_error)?; + if bytes.len() as u64 > MAX_JSON { + return Err("source journal exceeds 8 MiB".into()); + } + let mut temporary = tempfile::Builder::new().prefix(".pdfcraft-source-state-").tempfile_in(input).map_err(io_error)?; + temporary.write_all(&bytes).and_then(|_| temporary.as_file().sync_all()).map_err(io_error)?; + temporary.persist(&target).map_err(|error| io_error(error.error))?; + sync_directory(input) +} + +/// A synced temporary file does not make its renamed directory entry durable on Unix. +/// Confirm the journal/publication before permitting a destructive source action. +fn sync_directory(_path: &Path) -> Result<(), String> { + #[cfg(unix)] + File::open(_path) + .and_then(|directory| directory.sync_all()) + .map_err(|_| "cannot confirm source directory persistence; no further original-file action was performed".to_owned())?; + Ok(()) +} +fn read_bounded(path: &Path, limit: u64) -> Result, String> { + plain_file(path)?; + let mut bytes = Vec::new(); + File::open(path).map_err(io_error)?.take(limit.saturating_add(1)).read_to_end(&mut bytes).map_err(io_error)?; + if bytes.len() as u64 > limit { + return Err("source metadata exceeds its size limit".into()); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn long_names_keep_extensions_and_project_missing_entries_never_silently_disappear() { + let name = safe_filename(&format!("{}.PDF", "頁".repeat(200))); + assert!(name.len() <= 160); + assert!(name.ends_with(".pdf")); + assert!(relative_ok(Path::new(&name))); + let (root, config, inbox, originals) = fixture(SourceAction::Keep); + assert_eq!(local_paths(&config, Some(root.path())).unwrap().len(), 2); + fs::remove_file(originals.join("chapter/one.pdf")).unwrap(); + assert!(stage(&config, &inbox, Some(root.path()), &|| false, &CredentialStore::OsVault).unwrap_err().contains("missing, unprocessed")); + } + + #[test] + fn project_restart_preserves_manifest_order_for_missing_but_staged_originals() { + let (root, config, inbox, originals) = fixture(SourceAction::Keep); + let first = stage(&config, &inbox, Some(root.path()), &|| false, &CredentialStore::OsVault).unwrap(); + fs::remove_file(originals.join("chapter/one.pdf")).unwrap(); + let resumed = stage(&config, &inbox, Some(root.path()), &|| false, &CredentialStore::OsVault).unwrap(); + assert_eq!(first.items, resumed.items); + } + + #[test] + fn corrupt_receipts_cannot_repoint_a_confirmed_source() { + let (root, config, inbox, _) = fixture(SourceAction::Delete); + let report = stage(&config, &inbox, Some(root.path()), &|| false, &CredentialStore::OsVault).unwrap(); + let path = inbox.join(STATE_FILE); + let mut state: Journal = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + state.records.get_mut(&report.items[0].source_id).unwrap().remote.key.push_str("-different"); + fs::write(&path, serde_json::to_vec(&state).unwrap()).unwrap(); + assert!( + acknowledge(&config, &inbox, &report.items[0], &report.items[0].sha256, Some(root.path()), &|| false, &CredentialStore::OsVault).is_err() + ); + } + fn fixture(action: SourceAction) -> (tempfile::TempDir, SourceConfig, PathBuf, PathBuf) { + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("originals"); + let inbox = root.path().join("inbox"); + fs::create_dir(&source).unwrap(); + fs::create_dir(&inbox).unwrap(); + fs::create_dir(source.join("chapter")).unwrap(); + fs::write(source.join("chapter/one.pdf"), b"synthetic PDF source bytes").unwrap(); + fs::write(source.join("two.png"), b"synthetic image source bytes").unwrap(); + let manifest = source.join("project.pdfcraft.json"); + write_project_manifest( + &manifest, + &ProjectManifest { files: vec!["chapter/one.pdf".into(), "two.png".into()], ..Default::default() }, + Some(root.path()), + ) + .unwrap(); + (root, SourceConfig::Project(ProjectSource { manifest, action }), inbox, source) + } + #[test] + fn project_staging_is_atomic_durable_and_hash_bound_before_deleting_originals() { + let (root, config, inbox, originals) = fixture(SourceAction::Delete); + let store = CredentialStore::OsVault; + let report = stage(&config, &inbox, Some(root.path()), &|| false, &store).unwrap(); + assert_eq!(report.items.len(), 2); + assert_eq!(report.items[0].relative, PathBuf::from("chapter/one.pdf")); + assert!(originals.join("chapter/one.pdf").exists()); + let resumed = stage(&config, &inbox, Some(root.path()), &|| false, &store).unwrap(); + assert_eq!(report.items, resumed.items); + let first = &report.items[0]; + assert!(acknowledge(&config, &inbox, first, "wrong output input hash", Some(root.path()), &|| false, &store).is_err()); + assert!(originals.join("chapter/one.pdf").exists()); + acknowledge(&config, &inbox, first, &first.sha256, Some(root.path()), &|| false, &store).unwrap(); + assert!(!originals.join("chapter/one.pdf").exists()); + acknowledge(&config, &inbox, first, &first.sha256, Some(root.path()), &|| false, &store).unwrap(); + fs::remove_file(inbox.join(&first.staged_relative)).unwrap(); + assert_eq!(stage(&config, &inbox, Some(root.path()), &|| false, &store).unwrap().items.len(), 1); + } + #[test] + fn changed_originals_and_cancelled_operations_are_retained_and_recoverable() { + let (root, config, inbox, originals) = fixture(SourceAction::Delete); + let store = CredentialStore::OsVault; + assert!(stage(&config, &inbox, Some(root.path()), &|| true, &store).is_err()); + assert_eq!( + fs::read_dir(&inbox).unwrap().filter_map(Result::ok).filter(|item| item.path().extension().is_some_and(|ext| ext == "pdf")).count(), + 0 + ); + let report = stage(&config, &inbox, Some(root.path()), &|| false, &store).unwrap(); + let first = &report.items[0]; + fs::write(originals.join("chapter/one.pdf"), b"changed after OCR").unwrap(); + assert!(acknowledge(&config, &inbox, first, &first.sha256, Some(root.path()), &|| false, &store).is_err()); + assert!(originals.join("chapter/one.pdf").exists()); + let resumed = stage(&config, &inbox, Some(root.path()), &|| false, &store).unwrap(); + assert_eq!(resumed.pending_acknowledgements.len(), 1); + assert!(!resumed.items.iter().any(|item| item.source_id == first.source_id)); + } + #[test] + fn project_archive_is_no_clobber_and_configuration_and_path_escapes_fail() { + let (root, mut config, inbox, originals) = fixture(SourceAction::Keep); + let archive = root.path().join("archive"); + fs::create_dir(&archive).unwrap(); + if let SourceConfig::Project(source) = &mut config { + source.action = SourceAction::Move { destination: archive.to_string_lossy().into_owned() }; + } + let store = CredentialStore::OsVault; + let report = stage(&config, &inbox, Some(root.path()), &|| false, &store).unwrap(); + let item = &report.items[0]; + fs::write(archive.join(&item.staged_relative), b"unrelated archive").unwrap(); + assert!(acknowledge(&config, &inbox, item, &item.sha256, Some(root.path()), &|| false, &store).is_err()); + assert!(originals.join("chapter/one.pdf").exists()); + assert_eq!(fs::read(archive.join(&item.staged_relative)).unwrap(), b"unrelated archive"); + if let SourceConfig::Project(source) = &mut config { + source.action = SourceAction::Delete; + } + assert!(stage(&config, &inbox, Some(root.path()), &|| false, &store).is_err()); + assert!( + write_project_manifest( + &originals.join("bad.json"), + &ProjectManifest { files: vec!["../escape.pdf".into()], ..Default::default() }, + Some(root.path()) + ) + .is_err() + ); + assert!( + write_project_manifest( + &originals.join("abbyy.json"), + &ProjectManifest { format: "ABBYY".into(), files: vec!["two.png".into()], ..Default::default() }, + Some(root.path()) + ) + .is_err() + ); + } + struct Fake { + count: usize, + fail: bool, + } + impl Backend for Fake { + fn list(&mut self, _: &Control<'_>) -> Result, String> { + Ok((0..self.count) + .map(|i| RemoteFile { key: format!("{i}"), relative: format!("{i}.pdf").into(), revision: "1".into(), size: 3, group: None }) + .collect()) + } + fn read(&mut self, _: &RemoteFile, writer: &mut dyn Write, _: &Control<'_>) -> Result<(), String> { + writer.write_all(b"abc").unwrap(); + if self.fail { Err("network disconnected".into()) } else { Ok(()) } + } + fn finish(&mut self, _: &[Record], _: &SourceAction, _: &Control<'_>) -> Result<(), String> { + Ok(()) + } + } + #[test] + fn bounded_queues_signal_incomplete_batches_and_failed_downloads_never_publish() { + let dir = tempfile::tempdir().unwrap(); + let mut journal = Journal { config_hash: "test".into(), ..Default::default() }; + let mut fake = Fake { count: 1, fail: true }; + let control = Control::new(&|| false); + assert!(stage_with(&mut fake, &SourceConfig::Local, dir.path(), &mut journal, &control).is_err()); + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 0); + fake.fail = false; + fake.count = 129; + let report = stage_with(&mut fake, &SourceConfig::Local, dir.path(), &mut journal, &control).unwrap(); + assert_eq!(report.items.len(), 128); + assert!(report.has_more); + for record in journal.records.values_mut() { + record.ready = true; + } + let second = stage_with(&mut fake, &SourceConfig::Local, dir.path(), &mut journal, &control).unwrap(); + assert_eq!(second.items.len(), 1); + assert!(!second.has_more); + assert_eq!(second.pending_acknowledgements.len(), 128); + } + #[test] + fn task_json_and_debug_never_contain_credentials() { + let config = SourceConfig::Ftp(FtpSource { + host: "example.test".into(), + username: "scan-user".into(), + credential_id: "ftp-scanner".into(), + ..Default::default() + }); + let json = serde_json::to_string(&config).unwrap(); + assert!(!json.contains("password")); + assert!(!format!("{config:?}").contains("password")); + assert!(serde_json::from_str::(&json.replace("\"host\"", "\"password\":\"secret\",\"host\"")).is_err()); + assert!(matches!(serde_json::from_str::("{\"kind\":\"local\"}").unwrap(), SourceConfig::Local)); + } +} diff --git a/crates/engine/src/hot_folder_sources/credentials.rs b/crates/engine/src/hot_folder_sources/credentials.rs new file mode 100644 index 0000000000..f4df7ab872 --- /dev/null +++ b/crates/engine/src/hot_folder_sources/credentials.rs @@ -0,0 +1,618 @@ +//! Explicit protected service files or an explicitly selected per-user OS vault. +//! A permission failure never falls back to another store or unprotected files. +//! Chunking accommodates Windows Credential Manager's small per-entry blob limit. + +use serde::{Deserialize, Serialize}; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; +use zeroize::Zeroizing; + +const SERVICE: &str = "org.pdfcraft.hot-folder.sources"; +const MAX_SECRET: usize = 64 * 1024; +const CHUNK: usize = 1800; +static LOCK: OnceLock> = OnceLock::new(); + +/// Explicit execution context, never part of exported task JSON. Services use their +/// installer-created protected `secrets` directory; portable workers can select OsVault. +#[derive(Clone, Debug)] +pub enum CredentialStore { + OsVault, + ServiceDirectory { + directory: PathBuf, + }, + /// Explicit elevated provisioning, bound to an existing private service config. + /// Workers must run as the service account, never use this administrator context. + #[cfg(unix)] + ServiceAdministrator { + configuration: PathBuf, + owner_uid: u32, + owner_gid: u32, + }, +} +impl CredentialStore { + pub fn for_config(config: &Path, root: Option<&Path>) -> Result { + let parent = super::confined(config.parent().ok_or("configuration needs a parent folder")?, root)?; + let directory = parent.join("secrets"); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if rustix::process::geteuid().as_raw() == 0 { + let configuration = super::confined(config, root)?; + let metadata = + fs::symlink_metadata(&configuration).map_err(|_| "administrator provisioning needs an existing service configuration")?; + if metadata.uid() != 0 { + let policy = FilePolicy::Administrator { configuration: configuration.clone(), uid: metadata.uid(), gid: metadata.gid() }; + policy.validate_context()?; + if !directory.exists() { + use std::os::unix::fs::DirBuilderExt; + fs::DirBuilder::new().mode(0o700).create(&directory).map_err(|_| "cannot create protected service secrets directory")?; + policy.adopt(&File::open(&directory).map_err(|_| "cannot inspect new secrets directory")?)?; + } + policy.check(&directory, true)?; + return Ok(Self::ServiceAdministrator { configuration, owner_uid: metadata.uid(), owner_gid: metadata.gid() }); + } + } + } + // Unix: only the service account may provision this directory. Windows: the + // installer creates its protected ACL before anything can contain a secret. + #[cfg(unix)] + if !directory.exists() { + use std::os::unix::fs::DirBuilderExt; + fs::DirBuilder::new() + .mode(0o700) + .create(&directory) + .map_err(|_| "cannot create service secrets directory; provision as the configured service account")?; + } + check_private(&directory, true)?; + Ok(Self::ServiceDirectory { directory }) + } + fn vault(&self) -> Result, String> { + match self { + Self::OsVault => Ok(Box::new(OsVault)), + Self::ServiceDirectory { directory } => Ok(Box::new(FileVault::open(directory, FilePolicy::CurrentUser)?)), + #[cfg(unix)] + Self::ServiceAdministrator { configuration, owner_uid, owner_gid } => { + let directory = configuration.parent().ok_or("service configuration needs a parent")?.join("secrets"); + Ok(Box::new(FileVault::open( + &directory, + FilePolicy::Administrator { configuration: configuration.clone(), uid: *owner_uid, gid: *owner_gid }, + )?)) + } + } + } +} + +enum FilePolicy { + CurrentUser, + #[cfg(unix)] + Administrator { + configuration: PathBuf, + uid: u32, + gid: u32, + }, +} +impl FilePolicy { + fn validate_context(&self) -> Result<(), String> { + #[cfg(unix)] + if let Self::Administrator { configuration, uid, gid } = self { + if rustix::process::geteuid().as_raw() != 0 || *uid == 0 { + return Err("administrator credential provisioning requires root and a distinct non-root service owner".into()); + } + super::confined(configuration, None)?; + check_private_as(configuration, false, Some((*uid, *gid)))?; + check_private_as(configuration.parent().ok_or("service configuration needs a parent")?, true, Some((*uid, *gid)))?; + } + Ok(()) + } + fn check(&self, path: &Path, directory: bool) -> Result<(), String> { + self.validate_context()?; + #[cfg(unix)] + if let Self::Administrator { uid, gid, .. } = self { + return check_private_as(path, directory, Some((*uid, *gid))); + } + check_private(path, directory) + } + fn adopt(&self, _file: &File) -> Result<(), String> { + self.validate_context()?; + #[cfg(unix)] + if let Self::Administrator { uid, gid, .. } = self { + std::os::unix::fs::fchown(_file, Some(*uid), Some(*gid)) + .map_err(|_| "cannot assign new credential file to the configured service owner")?; + } + Ok(()) + } +} +struct FileVault { + directory: PathBuf, + policy: FilePolicy, + _lock: File, +} +impl FileVault { + fn open(directory: &Path, policy: FilePolicy) -> Result { + super::confined(directory, None)?; + policy.check(directory, true)?; + let path = directory.join(".credentials.lock"); + let mut options = fs::OpenOptions::new(); + options.create_new(true).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let lock = match options.open(&path) { + Ok(file) => { + policy.adopt(&file)?; + file + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + policy.check(&path, false)?; + fs::OpenOptions::new().read(true).write(true).open(&path).map_err(|_| "cannot open credential store lock")? + } + Err(_) => return Err("cannot create credential store lock".into()), + }; + policy.check(&path, false)?; + lock.try_lock().map_err(|_| "another process is refreshing or changing credentials; retry shortly")?; + Ok(FileVault { directory: directory.to_owned(), policy, _lock: lock }) + } + fn path(&self, name: &str) -> PathBuf { + self.directory.join(format!("{}.secret", super::hash(name.as_bytes()))) + } +} +impl Vault for FileVault { + fn put(&self, name: &str, bytes: &[u8]) -> Result<(), String> { + self.policy.check(&self.directory, true)?; + let target = self.path(name); + if fs::symlink_metadata(&target).is_ok() { + self.policy.check(&target, false)?; + } + let mut temporary = + tempfile::Builder::new().prefix(".credential-").tempfile_in(&self.directory).map_err(|_| "cannot create protected credential file")?; + self.policy.adopt(temporary.as_file())?; + // Verify before writing any secret, including inherited Windows ACLs. + self.policy.check(temporary.path(), false)?; + temporary.write_all(bytes).and_then(|_| temporary.as_file().sync_all()).map_err(|_| "cannot persist protected credential")?; + temporary.persist(target).map_err(|_| "cannot atomically replace protected credential")?; + Ok(()) + } + fn get(&self, name: &str) -> Result>>, String> { + self.policy.check(&self.directory, true)?; + let path = self.path(name); + match fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("cannot inspect protected credential".into()), + Ok(_) => {} + } + self.policy.check(&path, false)?; + let mut bytes = Zeroizing::new(Vec::new()); + File::open(path) + .map_err(|_| "cannot open protected credential")? + .take((MAX_SECRET + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| "cannot read protected credential")?; + if bytes.len() > MAX_SECRET { + return Err("protected credential exceeds its size limit".into()); + } + Ok(Some(bytes)) + } + fn remove(&self, name: &str) -> Result<(), String> { + self.policy.check(&self.directory, true)?; + let path = self.path(name); + match fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(_) => return Err("cannot inspect protected credential".into()), + Ok(_) => {} + } + self.policy.check(&path, false)?; + fs::remove_file(path).map_err(|_| "cannot remove protected credential".into()) + } +} + +fn check_private(path: &Path, directory: bool) -> Result<(), String> { + check_private_as(path, directory, None) +} +fn check_private_as(path: &Path, directory: bool, _expected_owner: Option<(u32, u32)>) -> Result<(), String> { + let metadata = + fs::symlink_metadata(path).map_err(|_| "protected secrets directory is unavailable; provision it as administrator/service account")?; + if super::is_link(&metadata) || (directory && !metadata.is_dir()) || (!directory && !metadata.is_file()) { + return Err("credential storage must use real directories and files, never links".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let (uid, gid) = _expected_owner.unwrap_or_else(|| (rustix::process::geteuid().as_raw(), metadata.gid())); + if metadata.uid() != uid || metadata.gid() != gid || metadata.mode() & 0o7777 != if directory { 0o700 } else { 0o600 } { + return Err( + "credentials require service-account ownership and mode 0700 for directories / 0600 for files; provision as that account".into() + ); + } + if !directory && metadata.nlink() != 1 { + return Err("credential files must not have hard links".into()); + } + } + #[cfg(windows)] + { + windows_private(path, directory)?; + } + Ok(()) +} + +#[cfg(windows)] +fn windows_private(path: &Path, directory: bool) -> Result<(), String> { + use windows_permissions::WindowsSecure; + use windows_permissions::constants::SecurityInformation; + let descriptor = path + .as_os_str() + .security_descriptor(SecurityInformation::Dacl | SecurityInformation::Owner) + .map_err(|_| "cannot inspect Windows credential ACL; run provisioning as administrator")?; + windows_descriptor_private(&descriptor, directory) +} + +#[cfg(windows)] +fn windows_descriptor_private(descriptor: &windows_permissions::SecurityDescriptor, directory: bool) -> Result<(), String> { + use windows_permissions::constants::AceType; + use windows_permissions::{LocalBox, Sid}; + let allowed: Vec> = ["S-1-5-18", "S-1-5-32-544", "S-1-5-19"] + .iter() + .map(|sid| sid.parse()) + .collect::>() + .map_err(|_| "cannot construct service ACL identities")?; + if descriptor.owner().is_none_or(|owner| !allowed.iter().any(|sid| owner == &**sid)) { + return Err("credential owner must be SYSTEM, Administrators or LocalService".into()); + } + let acl = descriptor.dacl().ok_or("credential ACL must not grant everyone access")?; + if acl.len() == 0 || acl.len() > 32 { + return Err("credential ACL is empty or unexpectedly complex".into()); + } + for index in 0..acl.len() { + let ace = acl.get_ace(index).ok_or("credential ACL is malformed")?; + if ace.ace_type() != AceType::ACCESS_ALLOWED_ACE_TYPE || ace.sid().is_none_or(|owner| !allowed.iter().any(|sid| owner == &**sid)) { + return Err("credential ACL may grant access only to SYSTEM, Administrators and LocalService; ordinary Users access is forbidden".into()); + } + } + if directory { + let sddl = descriptor.as_sddl().map_err(|_| "cannot inspect credential directory inheritance")?; + if !sddl.to_string_lossy().contains("D:P") { + return Err("credential directory must disable inherited ACLs; repair it using the installer".into()); + } + } + Ok(()) +} + +pub(super) fn valid_id(id: &str) -> Result<(), String> { + if id.is_empty() || id.len() > 128 || !id.bytes().all(|c| c.is_ascii_alphanumeric() || matches!(c, b'-' | b'_' | b'.')) { + return Err("credential ID must be 1..128 ASCII letters, digits, dots, underscores or hyphens".into()); + } + Ok(()) +} + +trait Vault { + fn put(&self, name: &str, bytes: &[u8]) -> Result<(), String>; + fn get(&self, name: &str) -> Result>>, String>; + fn remove(&self, name: &str) -> Result<(), String>; +} + +struct OsVault; + +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux", target_os = "freebsd", target_os = "openbsd"))] +impl Vault for OsVault { + fn put(&self, name: &str, bytes: &[u8]) -> Result<(), String> { + entry(name)?.set_secret(bytes).map_err(vault_error) + } + fn get(&self, name: &str) -> Result>>, String> { + match entry(name)?.get_secret() { + Ok(bytes) => Ok(Some(Zeroizing::new(bytes))), + Err(keyring::Error::NoEntry) => Ok(None), + Err(error) => Err(vault_error(error)), + } + } + fn remove(&self, name: &str) -> Result<(), String> { + match entry(name)?.delete_credential() { + Ok(()) | Err(keyring::Error::NoEntry) => Ok(()), + Err(error) => Err(vault_error(error)), + } + } +} + +#[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux", target_os = "freebsd", target_os = "openbsd")))] +impl Vault for OsVault { + fn put(&self, _: &str, _: &[u8]) -> Result<(), String> { + Err("OS credential storage is unavailable on this platform".into()) + } + fn get(&self, _: &str) -> Result>>, String> { + Err("OS credential storage is unavailable on this platform".into()) + } + fn remove(&self, _: &str) -> Result<(), String> { + Err("OS credential storage is unavailable on this platform".into()) + } +} + +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux", target_os = "freebsd", target_os = "openbsd"))] +fn entry(name: &str) -> Result { + keyring::Entry::new(SERVICE, name).map_err(vault_error) +} +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux", target_os = "freebsd", target_os = "openbsd"))] +fn vault_error(_: keyring::Error) -> String { + // Underlying platform errors can include secret attribute values; do not echo them. + "OS credential vault is locked, unavailable or access was denied; unlock the current user's vault and retry (no file fallback)".into() +} + +#[derive(Serialize, Deserialize)] +struct Pointer { + generation: String, + chunks: usize, + sha256: String, +} +fn pointer(vault: &dyn Vault, id: &str) -> Result, String> { + let Some(bytes) = vault.get(id)? else { + return Ok(None); + }; + if bytes.len() > 1024 { + return Err("credential manifest is corrupt".into()); + } + let value: Pointer = serde_json::from_slice(&bytes).map_err(|_| "credential manifest is corrupt".to_owned())?; + if value.generation.len() != 64 + || !value.generation.bytes().all(|c| c.is_ascii_hexdigit()) + || value.chunks == 0 + || value.chunks > MAX_SECRET.div_ceil(CHUNK) + { + return Err("credential manifest is corrupt".into()); + } + Ok(Some(value)) +} +fn chunk_name(id: &str, pointer: &Pointer, index: usize) -> String { + format!("{id}.{}.{}", pointer.generation, index) +} + +fn write(vault: &dyn Vault, id: &str, bytes: &[u8]) -> Result<(), String> { + valid_id(id)?; + if bytes.is_empty() || bytes.len() > MAX_SECRET { + return Err("credential size must be between 1 byte and 64 KiB".into()); + } + let old = pointer(vault, id)?; + // Content-addressing plus one atomic pointer update prevents a partial refresh from + // destroying the last usable credential. The hash is internal to the OS vault. + let generation = super::hash(bytes); + let next = Pointer { generation: generation.clone(), chunks: bytes.len().div_ceil(CHUNK), sha256: generation }; + if old.as_ref().is_some_and(|value| value.generation == next.generation) { + return Ok(()); + } + let mut stored = 0; + let result = (|| { + for (index, chunk) in bytes.chunks(CHUNK).enumerate() { + vault.put(&chunk_name(id, &next, index), chunk)?; + stored += 1; + } + vault.put(id, &serde_json::to_vec(&next).map_err(|_| "cannot encode credential manifest".to_owned())?) + })(); + if result.is_err() { + for index in 0..stored { + let _ = vault.remove(&chunk_name(id, &next, index)); + } + return result; + } + if let Some(old) = old { + for index in 0..old.chunks { + let _ = vault.remove(&chunk_name(id, &old, index)); + } + } + Ok(()) +} +fn read(vault: &dyn Vault, id: &str) -> Result>>, String> { + valid_id(id)?; + let Some(pointer) = pointer(vault, id)? else { + return Ok(None); + }; + let mut output = Zeroizing::new(Vec::new()); + for index in 0..pointer.chunks { + let bytes = vault.get(&chunk_name(id, &pointer, index))?.ok_or("credential store is incomplete; sign in again")?; + if bytes.len() > CHUNK || output.len().saturating_add(bytes.len()) > MAX_SECRET { + return Err("credential size limit exceeded".into()); + } + output.extend_from_slice(&bytes); + } + if super::hash(&output) != pointer.sha256 { + return Err("credential integrity check failed; sign in again".into()); + } + Ok(Some(output)) +} +fn delete(vault: &dyn Vault, id: &str) -> Result<(), String> { + valid_id(id)?; + if let Some(pointer) = pointer(vault, id)? { + // Remove the public lookup first; an interruption cannot expose a partially removed secret. + vault.remove(id)?; + for index in 0..pointer.chunks { + vault.remove(&chunk_name(id, &pointer, index))?; + } + } + Ok(()) +} +pub(super) fn put(store: &CredentialStore, id: &str, bytes: &[u8]) -> Result<(), String> { + let _lock = LOCK.get_or_init(|| Mutex::new(())).lock().map_err(|_| "credential store lock was interrupted")?; + write(store.vault()?.as_ref(), id, bytes) +} +pub(super) fn get(store: &CredentialStore, id: &str) -> Result>>, String> { + let _lock = LOCK.get_or_init(|| Mutex::new(())).lock().map_err(|_| "credential store lock was interrupted")?; + read(store.vault()?.as_ref(), id) +} + +pub fn credential_status(store: &CredentialStore, id: &str) -> Result { + Ok(get(store, id)?.is_some()) +} +pub fn forget_credentials(store: &CredentialStore, id: &str) -> Result<(), String> { + let _lock = LOCK.get_or_init(|| Mutex::new(())).lock().map_err(|_| "credential store lock was interrupted")?; + delete(store.vault()?.as_ref(), id) +} +pub fn save_ftp_password(store: &CredentialStore, id: &str, password: &str) -> Result<(), String> { + if password.contains(['\r', '\n', '\0']) || password.len() > 4096 { + return Err("FTP password contains a line break or exceeds 4096 bytes".into()); + } + // Prefix distinguishes FTP credentials from OAuth state if an ID is accidentally reused. + let payload = Zeroizing::new(format!("ftp-password-v1:{password}")); + put(store, id, payload.as_bytes()) +} +pub(super) fn ftp_password(store: &CredentialStore, id: &str) -> Result, String> { + let bytes = get(store, id)?.ok_or("FTP password is not stored; connect the source explicitly first")?; + let value = std::str::from_utf8(&bytes).map_err(|_| "FTP credential is corrupt")?; + let password = value.strip_prefix("ftp-password-v1:").ok_or("credential ID belongs to another source type")?; + Ok(Zeroizing::new(password.to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::{Cell, RefCell}; + use std::collections::BTreeMap; + #[derive(Default)] + struct Memory { + values: RefCell>>, + deny: Cell, + writes_left: Cell>, + } + impl Vault for Memory { + fn put(&self, name: &str, bytes: &[u8]) -> Result<(), String> { + if self.deny.get() || self.writes_left.get() == Some(0) { + return Err("permission denied".into()); + } + if let Some(left) = self.writes_left.get() { + self.writes_left.set(Some(left - 1)); + } + self.values.borrow_mut().insert(name.into(), bytes.to_vec()); + Ok(()) + } + fn get(&self, name: &str) -> Result>>, String> { + if self.deny.get() { + return Err("permission denied".into()); + } + Ok(self.values.borrow().get(name).cloned().map(Zeroizing::new)) + } + fn remove(&self, name: &str) -> Result<(), String> { + if self.deny.get() { + return Err("permission denied".into()); + } + self.values.borrow_mut().remove(name); + Ok(()) + } + } + #[test] + fn secret_chunks_round_trip_refresh_atomically_and_delete() { + let vault = Memory::default(); + let bytes = vec![42; 9000]; + write(&vault, "test", &bytes).unwrap(); + assert_eq!(*read(&vault, "test").unwrap().unwrap(), bytes); + vault.writes_left.set(Some(1)); + assert!(write(&vault, "test", &vec![9; 12000]).is_err()); + assert_eq!(*read(&vault, "test").unwrap().unwrap(), bytes); + vault.writes_left.set(None); + write(&vault, "test", b"refreshed token").unwrap(); + assert_eq!(*read(&vault, "test").unwrap().unwrap(), b"refreshed token"); + delete(&vault, "test").unwrap(); + assert!(vault.values.borrow().is_empty()); + } + #[test] + fn permission_denial_fails_closed_without_any_fallback() { + let vault = Memory::default(); + vault.deny.set(true); + assert!(write(&vault, "test", b"secret").is_err()); + assert!(read(&vault, "test").is_err()); + assert!(delete(&vault, "test").is_err()); + assert!(vault.values.borrow().is_empty()); + assert!(write(&vault, "../escape", b"secret").is_err()); + } + #[test] + fn broken_secret_chunk_is_detected() { + let vault = Memory::default(); + write(&vault, "test", b"secret").unwrap(); + let pointer = pointer(&vault, "test").unwrap().unwrap(); + vault.values.borrow_mut().insert(chunk_name("test", &pointer, 0), b"changed".to_vec()); + assert!(read(&vault, "test").is_err()); + } + #[cfg(windows)] + #[test] + fn windows_service_acl_accepts_only_protected_service_identities() { + use windows_permissions::{LocalBox, SecurityDescriptor}; + let good: LocalBox = "O:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LS)".parse().unwrap(); + assert!(windows_descriptor_private(&good, true).is_ok()); + for sddl in [ + "O:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LS)(A;OICI;GR;;;BU)", + "O:BAD:(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LS)", + "O:BUD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LS)", + "O:BAD:P(A;OICI;FA;;;WD)", + "O:BA", + ] { + let descriptor: LocalBox = sddl.parse().unwrap(); + assert!(windows_descriptor_private(&descriptor, true).is_err(), "{sddl}"); + } + let temp = tempfile::tempdir().unwrap(); + let store = CredentialStore::ServiceDirectory { directory: temp.path().to_owned() }; + assert!(save_ftp_password(&store, "test", "must not reach an ordinary temp directory").is_err()); + assert_eq!(fs::read_dir(temp.path()).unwrap().count(), 0); + } + #[cfg(unix)] + #[test] + fn unix_service_store_checks_modes_and_owner_and_round_trips_without_keyring() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + fs::set_permissions(temp.path(), fs::Permissions::from_mode(0o700)).unwrap(); + let store = CredentialStore::ServiceDirectory { directory: temp.path().to_owned() }; + save_ftp_password(&store, "test", "test-only password").unwrap(); + assert_eq!(*ftp_password(&store, "test").unwrap(), "test-only password"); + for file in fs::read_dir(temp.path()).unwrap() { + assert_eq!(file.unwrap().metadata().unwrap().permissions().mode() & 0o777, 0o600); + } + fs::set_permissions(temp.path(), fs::Permissions::from_mode(0o755)).unwrap(); + assert!(credential_status(&store, "test").is_err()); + fs::set_permissions(temp.path(), fs::Permissions::from_mode(0o700)).unwrap(); + forget_credentials(&store, "test").unwrap(); + assert!(!credential_status(&store, "test").unwrap()); + } + + #[cfg(unix)] + #[test] + fn administrator_context_rejects_unbound_or_nonprivate_service_config() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + fs::set_permissions(temp.path(), fs::Permissions::from_mode(0o700)).unwrap(); + let config = temp.path().join("tasks.json"); + fs::write(&config, b"{}").unwrap(); + fs::set_permissions(&config, fs::Permissions::from_mode(0o600)).unwrap(); + let store = CredentialStore::ServiceAdministrator { configuration: config.clone(), owner_uid: 65534, owner_gid: 65534 }; + assert!(save_ftp_password(&store, "test", "must not be written").is_err()); + assert!(!temp.path().join("secrets").exists()); + let zero = CredentialStore::ServiceAdministrator { configuration: config, owner_uid: 0, owner_gid: 0 }; + assert!(save_ftp_password(&zero, "test", "must not be written").is_err()); + } + + /// Run the compiled test executable via sudo, not cargo as root: this touches + /// only a fresh temporary directory and never connects to an account/keyring. + #[cfg(unix)] + #[test] + #[ignore = "requires root to test provisioning for a distinct synthetic service UID"] + fn unix_admin_provisioning_preserves_service_uid_gid_and_private_modes() { + use std::os::unix::fs::{MetadataExt, PermissionsExt, fchown}; + assert_eq!(rustix::process::geteuid().as_raw(), 0, "run only this ignored test executable with sudo"); + let temp = tempfile::tempdir().unwrap(); + fs::set_permissions(temp.path(), fs::Permissions::from_mode(0o700)).unwrap(); + let config = temp.path().join("tasks.json"); + fs::write(&config, b"{\"version\":1,\"tasks\":[]}").unwrap(); + fs::set_permissions(&config, fs::Permissions::from_mode(0o600)).unwrap(); + fchown(File::open(&config).unwrap(), Some(65534), Some(65534)).unwrap(); + fchown(File::open(temp.path()).unwrap(), Some(65534), Some(65534)).unwrap(); + let store = CredentialStore::for_config(&config, Some(temp.path())).unwrap(); + assert!(matches!(store, CredentialStore::ServiceAdministrator { owner_uid: 65534, owner_gid: 65534, .. })); + put(&store, "test", &vec![7; 9000]).unwrap(); + assert_eq!(*get(&store, "test").unwrap().unwrap(), vec![7; 9000]); + let directory = temp.path().join("secrets"); + assert_eq!(fs::metadata(&directory).unwrap().uid(), 65534); + for path in fs::read_dir(&directory).unwrap() { + let meta = path.unwrap().metadata().unwrap(); + assert_eq!((meta.uid(), meta.gid(), meta.mode() & 0o7777), (65534, 65534, 0o600)); + } + fs::set_permissions(&config, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(get(&store, "test").is_err()); + fs::set_permissions(&config, fs::Permissions::from_mode(0o600)).unwrap(); + forget_credentials(&store, "test").unwrap(); + assert!(!credential_status(&store, "test").unwrap()); + } +} diff --git a/crates/engine/src/hot_folder_sources/ftp.rs b/crates/engine/src/hot_folder_sources/ftp.rs new file mode 100644 index 0000000000..fba1a47282 --- /dev/null +++ b/crates/engine/src/hot_folder_sources/ftp.rs @@ -0,0 +1,704 @@ +//! Bounded passive FTP subset (RFC 959/2428/3659) and explicit TLS (RFC 4217). +//! Never uses active mode, follows server-supplied data hosts, or downgrades TLS. + +use super::*; +use rustls::{ClientConfig, ClientConnection, RootCertStore, StreamOwned}; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + mpsc, +}; + +const SOCKET_TIMEOUT: Duration = Duration::from_secs(10); +static RESOLVERS: AtomicUsize = AtomicUsize::new(0); + +pub(super) fn validate(source: &FtpSource) -> Result<(), String> { + if source.host.is_empty() || source.host.len() > 253 || source.host.contains(['/', '\\', '\r', '\n', '\0', '@', ' ']) || source.port == 0 { + return Err("FTP needs a hostname without URL/user information and a port from 1 to 65535".into()); + } + if source.username.is_empty() || source.username.len() > 256 || source.username.contains(['\r', '\n', '\0']) { + return Err("FTP username must be 1..256 bytes without line breaks".into()); + } + credentials::valid_id(&source.credential_id)?; + remote_path(&source.folder)?; + if let SourceAction::Move { destination } = &source.action { + remote_path(destination)?; + if destination == &source.folder || destination.starts_with(&format!("{}/", source.folder.trim_end_matches('/'))) { + return Err("FTP archive folder must be outside the watched folder".into()); + } + } + Ok(()) +} + +fn remote_path(value: &str) -> Result<(), String> { + if !value.starts_with('/') + || value.len() > 4096 + || value.contains(['\r', '\n', '\0', '\\']) + || value.split('/').any(|part| matches!(part, "." | "..")) + { + return Err("FTP folder must be an absolute slash path without traversal, backslashes or line breaks".into()); + } + Ok(()) +} + +enum Socket { + Plain(TcpStream), + Tls(Box>), +} + +/// rustls can perform several socket reads inside one public read/handshake. Keep +/// the whole-operation deadline on the underlying I/O too, including slow peers. +struct BoundedTcp { + socket: TcpStream, + deadline: Instant, +} +impl BoundedTcp { + fn timeout(&self) -> std::io::Result { + self.deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .map(|duration| duration.min(SOCKET_TIMEOUT)) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::TimedOut, "FTPS operation deadline exceeded")) + } +} +impl Read for BoundedTcp { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + self.socket.set_read_timeout(Some(self.timeout()?))?; + self.socket.read(bytes) + } +} +impl Write for BoundedTcp { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.socket.set_write_timeout(Some(self.timeout()?))?; + self.socket.write(bytes) + } + fn flush(&mut self) -> std::io::Result<()> { + self.socket.flush() + } +} +impl Read for Socket { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + match self { + Self::Plain(socket) => socket.read(bytes), + Self::Tls(socket) => socket.read(bytes), + } + } +} +impl Write for Socket { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + match self { + Self::Plain(socket) => socket.write(bytes), + Self::Tls(socket) => socket.write(bytes), + } + } + fn flush(&mut self) -> std::io::Result<()> { + match self { + Self::Plain(socket) => socket.flush(), + Self::Tls(socket) => socket.flush(), + } + } +} + +pub(super) struct Client { + source: FtpSource, + socket: Socket, + peer: SocketAddr, + tls: Option>, +} + +impl Client { + pub(super) fn connect(store: &CredentialStore, source: &FtpSource, control: &Control<'_>) -> Result { + validate(source)?; + let password = credentials::ftp_password(store, &source.credential_id)?; + Self::connect_password(source, &password, control) + } + fn connect_password(source: &FtpSource, password: &str, control: &Control<'_>) -> Result { + control.check()?; + let addresses = resolve(&source.host, source.port, control)?; + let mut connected = None; + for address in addresses.into_iter().take(8) { + control.check()?; + if let Ok(socket) = TcpStream::connect_timeout(&address, SOCKET_TIMEOUT) { + connected = Some(socket); + break; + } + } + let socket = connected.ok_or("FTP connection failed or timed out")?; + socket.set_read_timeout(Some(SOCKET_TIMEOUT)).and_then(|_| socket.set_write_timeout(Some(SOCKET_TIMEOUT))).map_err(io_error)?; + let peer = socket.peer_addr().map_err(io_error)?; + let mut client = Self { source: source.clone(), socket: Socket::Plain(socket), peer, tls: None }; + let welcome = client.reply(control)?.0; + if welcome != 220 { + return Err("FTP server did not provide a ready greeting".into()); + } + if source.security == FtpSecurity::ExplicitTls { + client.expect("AUTH TLS", &[234], control)?; + let mut roots = RootCertStore::empty(); + let certificates = rustls_native_certs::load_native_certs(); + for certificate in certificates.certs { + let _ = roots.add(certificate); + } + if roots.is_empty() { + return Err("no trusted OS certificate authorities are available for FTPS".into()); + } + let tls = ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) + .with_safe_default_protocol_versions() + .map_err(|_| "cannot configure FTPS protocol versions")? + .with_root_certificates(roots) + .with_no_client_auth(); + let tls = Arc::new(tls); + let Socket::Plain(socket) = client.socket else { + return Err("invalid FTP TLS state".into()); + }; + client.socket = secure(socket, &source.host, tls.clone(), control)?; + client.tls = Some(tls); + client.expect("PBSZ 0", &[200], control)?; + client.expect("PROT P", &[200], control)?; + } + let user = client.command(&format!("USER {}", source.username), control)?.0; + if user == 331 { + client.expect(&format!("PASS {password}"), &[230], control)?; + } else if user != 230 { + return Err("FTP login rejected; check the stored credentials".into()); + } + client.expect("TYPE I", &[200], control)?; + // UTF-8 negotiation is optional; listings themselves must decode as UTF-8. + let _ = client.command("OPTS UTF8 ON", control)?; + Ok(client) + } + + fn command(&mut self, command: &str, control: &Control<'_>) -> Result<(u16, String), String> { + control.check()?; + if command.contains(['\r', '\n', '\0']) || command.len() > 8192 { + return Err("FTP command contains an invalid path or credential".into()); + } + self.socket + .write_all(command.as_bytes()) + .and_then(|_| self.socket.write_all(b"\r\n")) + .and_then(|_| self.socket.flush()) + .map_err(|_| "FTP command failed or timed out".to_owned())?; + self.reply(control) + } + fn expect(&mut self, command: &str, expected: &[u16], control: &Control<'_>) -> Result { + let (status, text) = self.command(command, control)?; + if !expected.contains(&status) { + return Err(format!("FTP server rejected the operation (status {status}); original retained")); + } + Ok(text) + } + fn reply(&mut self, control: &Control<'_>) -> Result<(u16, String), String> { + let mut text = String::new(); + let mut first = None; + for _ in 0..128 { + control.check()?; + let mut line = Vec::new(); + for _ in 0..4096 { + control.check()?; + let mut byte = [0_u8]; + self.socket.read_exact(&mut byte).map_err(|_| "FTP response failed or timed out".to_owned())?; + line.extend_from_slice(&byte); + if byte.first() == Some(&b'\n') { + break; + } + } + if line.last() != Some(&b'\n') || text.len().saturating_add(line.len()) > 64 * 1024 { + return Err("FTP response exceeds the size limit".into()); + } + let line = std::str::from_utf8(&line).map_err(|_| "FTP response is not UTF-8")?; + let code = line.get(..3).and_then(|value| value.parse::().ok()); + text.push_str(line); + if first.is_none() { + let code = code.ok_or("FTP response has no status code")?; + if line.as_bytes().get(3) == Some(&b' ') { + return Ok((code, text)); + } + if line.as_bytes().get(3) != Some(&b'-') { + return Err("invalid FTP multiline response".into()); + } + first = Some(code); + } else if code == first && line.as_bytes().get(3) == Some(&b' ') { + return Ok((code.ok_or("missing FTP response code")?, text)); + } + } + Err("FTP response has too many lines".into()) + } + fn passive(&mut self, control: &Control<'_>) -> Result { + let (status, response) = self.command("EPSV", control)?; + let port = if status == 229 { + epsv_port(&response)? + } else if matches!(status, 500 | 501 | 502 | 504 | 522) { + let response = self.expect("PASV", &[227], control)?; + pasv_port(&response)? + } else { + return Err(format!("FTP passive mode rejected (status {status})")); + }; + let address = SocketAddr::new(self.peer.ip(), port); + let socket = TcpStream::connect_timeout(&address, SOCKET_TIMEOUT).map_err(|_| "FTP data connection failed or timed out")?; + socket.set_read_timeout(Some(SOCKET_TIMEOUT)).and_then(|_| socket.set_write_timeout(Some(SOCKET_TIMEOUT))).map_err(io_error)?; + Ok(socket) + } + fn transfer(&mut self, command: &str, writer: &mut dyn Write, limit: u64, control: &Control<'_>) -> Result { + let socket = self.passive(control)?; + self.expect(command, &[125, 150], control)?; + let mut data = if let Some(tls) = &self.tls { secure(socket, &self.source.host, tls.clone(), control)? } else { Socket::Plain(socket) }; + let count = copy_limited(&mut data, writer, limit, control)?; + drop(data); + if !matches!(self.reply(control)?.0, 226 | 250) { + return Err("FTP transfer did not finish successfully".into()); + } + Ok(count) + } + fn listing(&mut self, folder: &str, control: &Control<'_>) -> Result, String> { + remote_path(folder)?; + let mut bytes = Vec::new(); + self.transfer(&format!("MLSD {folder}"), &mut bytes, MAX_JSON, control)?; + parse_listing(&bytes) + } + fn metadata(&mut self, path: &str, control: &Control<'_>) -> Result, String> { + remote_path(path)?; + let (parent, name) = path.rsplit_once('/').ok_or("invalid remote path")?; + Ok(self.listing(if parent.is_empty() { "/" } else { parent }, control)?.into_iter().find(|entry| entry.name == name && !entry.directory)) + } + fn verify(&mut self, record: &Record, control: &Control<'_>) -> Result { + let Some(meta) = self.metadata(&record.remote.key, control)? else { + return Ok(false); + }; + if meta.size != record.remote.size || meta.revision != record.remote.revision { + return Err("FTP original changed after processing; it was not moved or deleted".into()); + } + let mut sink = HashWriter::default(); + self.transfer(&format!("RETR {}", record.remote.key), &mut sink, record.remote.size, control)?; + if sink.finish() != record.source.sha256 { + return Err("FTP original content changed after processing; it was not moved or deleted".into()); + } + Ok(true) + } +} + +impl Backend for Client { + fn list(&mut self, control: &Control<'_>) -> Result, String> { + let mut pending = vec![(self.source.folder.clone(), PathBuf::new(), 0_usize)]; + let mut result = Vec::new(); + let mut count = 0_usize; + while let Some((folder, relative, depth)) = pending.pop() { + control.check()?; + let mut entries = self.listing(&folder, control)?; + entries.sort_by(|left, right| left.name.cmp(&right.name)); + count = count.saturating_add(entries.len()); + if count > MAX_ITEMS { + return Err("FTP listing exceeds 10000 entries".into()); + } + for entry in entries { + let path = format!("{}/{}", folder.trim_end_matches('/'), entry.name); + let relative = relative.join(&entry.name); + if entry.directory && self.source.recursive { + if depth >= 16 { + return Err("FTP folder nesting exceeds 16 levels".into()); + } + pending.push((path, relative, depth + 1)); + } else if !entry.directory && supported(&relative) { + result.push(RemoteFile { key: path, relative, revision: entry.revision, size: entry.size, group: None }); + } + } + } + result.sort_by(|left, right| left.relative.cmp(&right.relative)); + Ok(result) + } + fn read(&mut self, file: &RemoteFile, writer: &mut dyn Write, control: &Control<'_>) -> Result<(), String> { + let meta = self.metadata(&file.key, control)?.ok_or("FTP source disappeared before download")?; + if meta.size != file.size || meta.revision != file.revision { + return Err("FTP source changed before download; retry after it settles".into()); + } + let size = self.transfer(&format!("RETR {}", file.key), writer, file.size, control)?; + let meta = self.metadata(&file.key, control)?.ok_or("FTP source disappeared during download")?; + if size != file.size || meta.size != file.size || meta.revision != file.revision { + return Err("FTP source changed during download; partial download was discarded".into()); + } + Ok(()) + } + fn finish(&mut self, files: &[Record], action: &SourceAction, control: &Control<'_>) -> Result<(), String> { + for record in files { + control.check()?; + if !record.remote.key.starts_with(&format!("{}/", self.source.folder.trim_end_matches('/'))) { + return Err("FTP receipt does not belong to the watched folder".into()); + } + match action { + SourceAction::Keep => {} + SourceAction::Delete => { + // Missing source is an idempotent success after an interrupted delete. + if self.verify(record, control)? { + self.expect(&format!("DELE {}", record.remote.key), &[250], control)?; + } + } + SourceAction::Move { destination } => { + remote_path(destination)?; + // Stable opaque prefix avoids clobbering an unrelated archive filename. + let target = format!("{}/{}", destination.trim_end_matches('/'), record.source.staged_relative.to_string_lossy()); + if let Some(existing) = self.metadata(&target, control)? { + let mut sink = HashWriter::default(); + if existing.size != record.remote.size { + return Err("FTP archive filename is occupied; original retained".into()); + } + self.transfer(&format!("RETR {target}"), &mut sink, record.remote.size, control)?; + if sink.finish() == record.source.sha256 && self.metadata(&record.remote.key, control)?.is_none() { + continue; + } + return Err("FTP archive filename is occupied; original retained".into()); + } + if !self.verify(record, control)? { + return Err("FTP original and verified archive are both missing".into()); + } + self.expect(&format!("RNFR {}", record.remote.key), &[350], control)?; + self.expect(&format!("RNTO {target}"), &[250], control)?; + } + } + } + Ok(()) + } +} + +#[derive(Default)] +struct HashWriter(Sha256); +impl Write for HashWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +impl HashWriter { + fn finish(self) -> String { + self.0.finalize().iter().map(|byte| format!("{byte:02x}")).collect() + } +} + +fn secure(socket: TcpStream, host: &str, config: Arc, control: &Control<'_>) -> Result { + let name = rustls::pki_types::ServerName::try_from(host.to_owned()).map_err(|_| "invalid FTPS server name")?; + let mut connection = ClientConnection::new(config, name).map_err(|_| "cannot initialize FTPS")?; + let mut socket = BoundedTcp { socket, deadline: control.deadline }; + while connection.is_handshaking() { + control.check()?; + connection + .complete_io(&mut socket) + .map_err(|_| "FTPS TLS handshake failed: check the server name, trusted certificate and network (TLS was not downgraded)")?; + } + Ok(Socket::Tls(Box::new(StreamOwned::new(connection, socket)))) +} +fn resolve(host: &str, port: u16, control: &Control<'_>) -> Result, String> { + if RESOLVERS.fetch_add(1, Ordering::SeqCst) >= 4 { + RESOLVERS.fetch_sub(1, Ordering::SeqCst); + return Err("too many pending FTP hostname lookups; retry later".into()); + } + let host = host.to_owned(); + let (sender, receiver) = mpsc::sync_channel(1); + let spawned = std::thread::Builder::new().name("pdfcraft-ftp-dns".into()).spawn(move || { + let result = (host.as_str(), port).to_socket_addrs().map(|addresses| addresses.take(8).collect::>()); + let _ = sender.send(result); + RESOLVERS.fetch_sub(1, Ordering::SeqCst); + }); + if spawned.is_err() { + RESOLVERS.fetch_sub(1, Ordering::SeqCst); + return Err("cannot start FTP hostname lookup".into()); + } + let deadline = Instant::now() + SOCKET_TIMEOUT; + loop { + control.check()?; + if Instant::now() > deadline { + return Err("FTP hostname lookup timed out".into()); + } + match receiver.recv_timeout(Duration::from_millis(100)) { + Ok(Ok(addresses)) => return Ok(addresses), + Ok(Err(_)) | Err(mpsc::RecvTimeoutError::Disconnected) => return Err("FTP hostname lookup failed".into()), + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } +} +fn parenthesized(value: &str) -> Result<&str, String> { + value.rsplit_once('(').and_then(|(_, tail)| tail.split_once(')')).map(|(inner, _)| inner).ok_or_else(|| "invalid FTP passive response".into()) +} +fn epsv_port(value: &str) -> Result { + let value = parenthesized(value)?; + let delimiter = value.chars().next().ok_or("invalid FTP extended passive response")?; + let parts = value.split(delimiter).collect::>(); + if parts.len() != 5 || parts.first() != Some(&"") || parts.get(1) != Some(&"") || parts.get(2) != Some(&"") || parts.get(4) != Some(&"") { + return Err("invalid FTP extended passive response".into()); + } + let port = parts.get(3).and_then(|value| value.parse::().ok()).filter(|port| *port > 0).ok_or("invalid FTP data port")?; + Ok(port) +} +fn pasv_port(value: &str) -> Result { + let parts = parenthesized(value)?.split(',').map(str::parse::).collect::, _>>().map_err(|_| "invalid FTP passive response")?; + if parts.len() != 6 { + return Err("invalid FTP passive response".into()); + } + let port = u16::from(*parts.get(4).ok_or("invalid FTP passive port")?) * 256 + u16::from(*parts.get(5).ok_or("invalid FTP passive port")?); + if port == 0 { + return Err("invalid FTP passive port".into()); + } + Ok(port) +} +struct Entry { + name: String, + directory: bool, + size: u64, + revision: String, +} +fn parse_listing(bytes: &[u8]) -> Result, String> { + let text = std::str::from_utf8(bytes).map_err(|_| "FTP listing is not UTF-8; this source requires RFC 3659 MLSD")?; + let mut result = Vec::new(); + for line in text.lines() { + if line.is_empty() { + continue; + } + if result.len() >= MAX_ITEMS || line.len() > 8192 { + return Err("FTP listing exceeds its entry/line limit".into()); + } + let (facts, name) = line.split_once(' ').ok_or("FTP server must support RFC 3659 MLSD listings")?; + let facts = facts + .split(';') + .filter_map(|fact| fact.split_once('=')) + .map(|(key, value)| (key.to_ascii_lowercase(), value)) + .collect::>(); + let kind = facts.get("type").copied().unwrap_or_default(); + if matches!(kind, "cdir" | "pdir") || kind.to_ascii_lowercase().contains("slink") { + continue; + } + if !matches!(kind, "file" | "dir") { + continue; + } + if name.is_empty() || name.contains(['/', '\\', '\r', '\n', '\0']) || matches!(name, "." | "..") { + return Err("FTP listing contains an unsafe filename".into()); + } + let directory = kind == "dir"; + let size = facts.get("size").and_then(|value| value.parse().ok()).unwrap_or(0); + let modified = facts.get("modify").copied().unwrap_or_default(); + if !directory && (modified.len() < 14 || modified.len() > 24 || !modified.chars().all(|c| c.is_ascii_digit() || c == '.')) { + return Err("FTP file is missing its MLSD modification time; stable version detection is unavailable".into()); + } + let unique = facts.get("unique").copied().unwrap_or_default(); + result.push(Entry { name: name.into(), directory, size, revision: format!("{modified}:{unique}") }); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{BufRead, BufReader}; + use std::net::TcpListener; + use std::sync::Mutex; + + type ServerFiles = Arc>>>; + + #[test] + fn real_tls_verifies_trust_and_hostname_and_enforces_deadline() { + let generated = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]).unwrap(); + let cert = generated.cert.der().clone(); + let server = rustls::ServerConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![cert.clone()], rustls::pki_types::PrivatePkcs8KeyDer::from(generated.signing_key.serialize_der()).into()) + .unwrap(); + let server = Arc::new(server); + for (trust, host, succeeds) in [(true, "localhost", true), (false, "localhost", false), (true, "other.example", false)] { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let config = server.clone(); + let thread = std::thread::spawn(move || { + let (socket, _) = listener.accept().unwrap(); + socket.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + socket.set_write_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut tls = StreamOwned::new(rustls::ServerConnection::new(config).unwrap(), socket); + let mut bytes = [0_u8; 4]; + if tls.read_exact(&mut bytes).is_ok() { + assert_eq!(&bytes, b"ping"); + tls.write_all(b"pong").unwrap(); + tls.flush().unwrap(); + } + }); + let mut roots = RootCertStore::empty(); + if trust { + roots.add(cert.clone()).unwrap(); + } + let config = ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + let socket = TcpStream::connect(address).unwrap(); + let mut result = secure(socket, host, Arc::new(config), &Control::new(&|| false)); + assert_eq!(result.is_ok(), succeeds); + if let Ok(socket) = &mut result { + socket.write_all(b"ping").unwrap(); + let mut bytes = [0_u8; 4]; + socket.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"pong"); + } + drop(result); + thread.join().unwrap(); + } + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut socket = BoundedTcp { socket: TcpStream::connect(listener.local_addr().unwrap()).unwrap(), deadline: Instant::now() }; + assert_eq!(socket.read(&mut [0_u8]).unwrap_err().kind(), std::io::ErrorKind::TimedOut); + } + + fn fake_server() -> (FtpSource, ServerFiles, std::thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let files = Arc::new(Mutex::new(BTreeMap::from([("/incoming/scan.pdf".to_owned(), b"synthetic PDF bytes".to_vec())]))); + let remote = files.clone(); + let thread = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + socket.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut reader = BufReader::new(socket.try_clone().unwrap()); + socket.write_all(b"220 Synthetic FTP fixture\r\n").unwrap(); + let mut data_listener = None; + let mut rename = String::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let command = line.trim_end_matches(['\r', '\n']); + let (verb, argument) = command.split_once(' ').unwrap_or((command, "")); + match verb { + "USER" => socket.write_all(b"331 Password required\r\n").unwrap(), + "PASS" => socket.write_all(b"230 Logged in\r\n").unwrap(), + "TYPE" | "OPTS" => socket.write_all(b"200 Accepted\r\n").unwrap(), + "EPSV" => { + let data = TcpListener::bind("127.0.0.1:0").unwrap(); + write!(socket, "229 Entering (|||{}|)\r\n", data.local_addr().unwrap().port()).unwrap(); + data_listener = Some(data); + } + "MLSD" | "RETR" => { + socket.write_all(b"150 Data follows\r\n").unwrap(); + let (mut data, _) = data_listener.take().unwrap().accept().unwrap(); + if verb == "MLSD" { + for (path, bytes) in remote.lock().unwrap().iter() { + let (parent, name) = path.rsplit_once('/').unwrap(); + if parent == argument.trim_end_matches('/') { + write!(data, "type=file;size={};modify=20261011000000; {name}\r\n", bytes.len()).unwrap(); + } + } + } else { + data.write_all(remote.lock().unwrap().get(argument).unwrap()).unwrap(); + } + drop(data); + socket.write_all(b"226 Complete\r\n").unwrap(); + } + "DELE" => { + remote.lock().unwrap().remove(argument); + socket.write_all(b"250 Deleted\r\n").unwrap(); + } + "RNFR" => { + rename = argument.into(); + socket.write_all(b"350 Rename target required\r\n").unwrap(); + } + "RNTO" => { + let mut files = remote.lock().unwrap(); + let bytes = files.remove(&rename).unwrap(); + files.insert(argument.into(), bytes); + socket.write_all(b"250 Renamed\r\n").unwrap(); + } + "AUTH" => { + socket.write_all(b"534 TLS unavailable\r\n").unwrap(); + } + _ => socket.write_all(b"502 Unsupported\r\n").unwrap(), + } + } + }); + ( + FtpSource { + host: address.ip().to_string(), + port: address.port(), + security: FtpSecurity::Plain, + username: "fixture".into(), + credential_id: "fixture".into(), + folder: "/incoming".into(), + ..Default::default() + }, + files, + thread, + ) + } + + #[test] + fn real_passive_ftp_socket_stages_and_verifies_before_idempotent_move() { + let (source, remote, thread) = fake_server(); + let control = Control::new(&|| false); + let mut client = Client::connect_password(&source, "fixture password", &control).unwrap(); + let file = client.list(&control).unwrap().remove(0); + let mut bytes = Vec::new(); + client.read(&file, &mut bytes, &control).unwrap(); + assert_eq!(bytes, b"synthetic PDF bytes"); + let record = Record { + source: StagedSource { + source_id: "fixture".into(), + relative: "scan.pdf".into(), + staged_relative: "stable-scan.pdf".into(), + sha256: hash(&bytes), + }, + remote: file, + ready: true, + acknowledged: false, + }; + assert!(remote.lock().unwrap().contains_key("/incoming/scan.pdf")); + client.finish(std::slice::from_ref(&record), &SourceAction::Move { destination: "/archive".into() }, &control).unwrap(); + assert!(!remote.lock().unwrap().contains_key("/incoming/scan.pdf")); + assert_eq!(remote.lock().unwrap().get("/archive/stable-scan.pdf").unwrap(), b"synthetic PDF bytes"); + client.finish(&[record], &SourceAction::Move { destination: "/archive".into() }, &control).unwrap(); + drop(client); + thread.join().unwrap(); + } + #[test] + fn ftps_never_falls_back_to_plain_ftp_and_changed_remote_data_is_retained() { + let (mut source, _, thread) = fake_server(); + source.security = FtpSecurity::ExplicitTls; + assert!(Client::connect_password(&source, "fixture", &Control::new(&|| false)).is_err()); + thread.join().unwrap(); + let (source, remote, thread) = fake_server(); + let control = Control::new(&|| false); + let mut client = Client::connect_password(&source, "fixture", &control).unwrap(); + let file = client.list(&control).unwrap().remove(0); + let record = Record { + source: StagedSource { + source_id: "fixture".into(), + relative: "scan.pdf".into(), + staged_relative: "stable-scan.pdf".into(), + sha256: hash(b"synthetic PDF bytes"), + }, + remote: file, + ready: true, + acknowledged: false, + }; + remote.lock().unwrap().insert("/incoming/scan.pdf".into(), b"different PDF bytes".to_vec()); + assert!(client.finish(&[record], &SourceAction::Delete, &control).is_err()); + assert!(remote.lock().unwrap().contains_key("/incoming/scan.pdf")); + drop(client); + thread.join().unwrap(); + } + #[test] + fn passive_ports_never_use_the_server_supplied_host() { + assert_eq!(epsv_port("229 Entering (|||12345|)\r\n").unwrap(), 12345); + assert_eq!(pasv_port("227 Entering (10,0,0,1,4,1)\r\n").unwrap(), 1025); + for value in ["(|||0|)", "(|||65536|)", "(|||1|x)", "no tuple"] { + assert!(epsv_port(value).is_err()); + } + } + #[test] + fn listing_requires_stable_versions_and_rejects_paths_and_links() { + let entries = + parse_listing(b"type=file;size=123;modify=20261011000000;unique=x; scan.pdf\r\ntype=dir; sub\r\ntype=OS.unix=slink; ignore.pdf\r\n") + .unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].size, 123); + assert_eq!(entries[0].revision, "20261011000000:x"); + assert!(parse_listing(b"type=file;size=1; missing.pdf\r\n").is_err()); + assert!(parse_listing(b"type=file;size=1;modify=20261011000000; ../bad.pdf\r\n").is_err()); + assert!(remote_path("/safe/../bad").is_err()); + } +} diff --git a/crates/engine/src/hot_folder_sources/outlook.rs b/crates/engine/src/hot_folder_sources/outlook.rs new file mode 100644 index 0000000000..37c3756c0e --- /dev/null +++ b/crates/engine/src/hot_folder_sources/outlook.rs @@ -0,0 +1,786 @@ +//! Microsoft Graph delegated device-code flow. Tokens/device secrets stay in the selected protected store. +//! Global Microsoft cloud endpoints are fixed; pagination cannot redirect credentials. + +use super::*; +use serde_json::{Value, json}; +use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; + +const GRAPH: &str = "https://graph.microsoft.com/v1.0"; +const LOGIN: &str = "https://login.microsoftonline.com"; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DeviceSignIn { + pub flow_id: String, + pub user_code: String, + pub verification_uri: String, + pub expires_at: u64, + pub interval_seconds: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum SignInStatus { + Pending { retry_after_seconds: u64 }, + Authorized, + Expired, + Declined, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MailFolder { + pub id: String, + pub name: String, + pub child_count: u64, +} + +#[derive(Serialize, Deserialize, Zeroize, ZeroizeOnDrop)] +struct OAuthSecret { + binding: String, + access_token: String, + refresh_token: String, + expires_at: u64, + account_id: String, + scope: String, +} +#[derive(Serialize, Deserialize, Zeroize, ZeroizeOnDrop)] +struct DeviceSecret { + binding: String, + device_code: String, + expires_at: u64, + next_poll: u64, + interval: u64, +} + +pub(super) fn validate(source: &OutlookSource) -> Result<(), String> { + credentials::valid_id(&source.credential_id)?; + if source.client_id.len() != 36 + || !source.client_id.bytes().enumerate().all(|(index, c)| if matches!(index, 8 | 13 | 18 | 23) { c == b'-' } else { c.is_ascii_hexdigit() }) + { + return Err("Outlook needs your own Microsoft Entra Application (client) ID in GUID form; enable public-client/device-code flow".into()); + } + if source.tenant.is_empty() || source.tenant.len() > 253 || !source.tenant.bytes().all(|c| c.is_ascii_alphanumeric() || matches!(c, b'-' | b'.')) + { + return Err("Outlook tenant must be common, consumers, organizations, a tenant GUID or tenant domain".into()); + } + valid_graph_id(&source.folder_id)?; + if let SourceAction::Move { destination } = &source.action { + valid_graph_id(destination)?; + if destination == &source.folder_id { + return Err("Outlook archive folder must differ from the watched mail folder".into()); + } + } + Ok(()) +} +fn valid_graph_id(id: &str) -> Result<(), String> { + if id.is_empty() || id.len() > 2048 || matches!(id, "." | "..") || id.chars().any(char::is_control) { + return Err("Outlook resource ID must be 1..2048 characters without control characters".into()); + } + Ok(()) +} +fn scope(source: &OutlookSource) -> &'static str { + if matches!(source.action, SourceAction::Keep) { "offline_access User.Read Mail.Read" } else { "offline_access User.Read Mail.ReadWrite" } +} +fn binding(source: &OutlookSource) -> String { + hash(format!("{}\n{}\n{}", source.client_id, source.tenant, scope(source)).as_bytes()) +} +fn auth_url(source: &OutlookSource, endpoint: &str) -> String { + format!("{LOGIN}/{}/oauth2/v2.0/{endpoint}", source.tenant) +} +fn form(fields: &[(&str, &str)]) -> Zeroizing { + Zeroizing::new(url::form_urlencoded::Serializer::new(String::new()).extend_pairs(fields.iter().copied()).finish()) +} +fn graph_url(parts: &[&str]) -> Result { + let mut url = url::Url::parse(GRAPH).map_err(|_| "invalid Graph base URL")?; + { + let mut segments = url.path_segments_mut().map_err(|_| "invalid Graph base path")?; + for part in parts { + valid_graph_id(part)?; + segments.push(part); + } + } + Ok(url.into()) +} +fn graph_url_ok(value: &str) -> Result<(), String> { + let url = url::Url::parse(value).map_err(|_| "invalid Outlook pagination URL")?; + if url.scheme() != "https" + || url.host_str() != Some("graph.microsoft.com") + || url.port().is_some_and(|port| port != 443) + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + || !url.path().starts_with("/v1.0/me/") + { + return Err("Outlook pagination URL left the permitted Microsoft Graph mailbox endpoint".into()); + } + Ok(()) +} + +struct Response { + status: u16, + body: Zeroizing>, + retry_after: u64, +} +impl Response { + fn json(&self) -> Result { + serde_json::from_slice(&self.body).map_err(|_| "Microsoft returned malformed or oversized JSON".into()) + } + fn success(&self) -> Result<(), String> { + match self.status { + 200..=299 => Ok(()), + 401 => Err("Outlook authorization expired or was revoked; sign in again".into()), + 403 => Err("Outlook permission denied; sign in with the required mail scope or check the organization's policy".into()), + 429 => Err(format!("Microsoft Graph throttled this source; retry after at least {} seconds", self.retry_after.max(1))), + 404 => Err("Outlook folder, message or attachment was not found".into()), + 500..=599 => Err("Microsoft Graph is temporarily unavailable; retry later".into()), + _ => Err(format!("Microsoft request failed (HTTP {}); originals retained", self.status)), + } + } +} + +trait Http { + #[allow(clippy::too_many_arguments)] + fn request( + &self, + method: &str, + url: &str, + bearer: Option<&str>, + body: Option<&str>, + is_form: bool, + limit: u64, + control: &Control<'_>, + ) -> Result; +} +struct Network(ureq::Agent); +impl Network { + fn new() -> Result { + let certs = rustls_native_certs::load_native_certs().certs; + let certs = certs.iter().map(|cert| ureq::tls::Certificate::from_der(cert.as_ref()).to_owned()).collect::>(); + if certs.is_empty() { + return Err("no trusted OS certificate authorities are available for Outlook".into()); + } + let agent = ureq::Agent::config_builder() + .https_only(true) + .max_redirects(0) + .http_status_as_error(false) + .timeout_global(Some(Duration::from_secs(45))) + .tls_config(ureq::tls::TlsConfig::builder().root_certs(ureq::tls::RootCerts::new_with_certs(&certs)).build()) + .build() + .new_agent(); + Ok(Self(agent)) + } +} +impl Http for Network { + fn request( + &self, + method: &str, + url: &str, + bearer: Option<&str>, + body: Option<&str>, + is_form: bool, + limit: u64, + control: &Control<'_>, + ) -> Result { + control.check()?; + let auth = Zeroizing::new(bearer.map(|value| format!("Bearer {value}")).unwrap_or_default()); + let mut response = match method { + "GET" => self.0.get(url).header("Authorization", auth.as_str()).header("Prefer", "IdType=\"ImmutableId\"").call(), + "DELETE" => self.0.delete(url).header("Authorization", auth.as_str()).header("Prefer", "IdType=\"ImmutableId\"").call(), + "POST" => self + .0 + .post(url) + .header("Authorization", auth.as_str()) + .header("Prefer", "IdType=\"ImmutableId\"") + .header("Content-Type", if is_form { "application/x-www-form-urlencoded" } else { "application/json" }) + .send(body.unwrap_or_default()), + _ => return Err("unsupported Outlook request method".into()), + } + .map_err(|_| "Microsoft connection failed, timed out or failed TLS verification; check the network and retry".to_owned())?; + let status = response.status().as_u16(); + let retry_after = + response.headers().get("Retry-After").and_then(|value| value.to_str().ok()).and_then(|value| value.parse().ok()).unwrap_or(60); + let mut bytes = Zeroizing::new(Vec::new()); + copy_limited(&mut response.body_mut().as_reader(), &mut *bytes, limit, control)?; + Ok(Response { status, body: bytes, retry_after }) + } +} + +pub fn outlook_begin_sign_in(store: &CredentialStore, source: &OutlookSource) -> Result { + guarded(|| { + validate(source)?; + let control = Control::new(&|| false); + let network = Network::new()?; + let body = form(&[("client_id", &source.client_id), ("scope", scope(source))]); + let response = network.request("POST", &auth_url(source, "devicecode"), None, Some(&body), true, MAX_JSON, &control)?; + response.success()?; + let data = response.json()?; + let (public, secret) = parse_device(source, &data, crate::hot_folder::now())?; + let encoded = Zeroizing::new(serde_json::to_vec(&secret).map_err(|_| "cannot encode sign-in state")?); + credentials::put(store, &public.flow_id, &encoded)?; + Ok(public) + }) +} +fn parse_device(source: &OutlookSource, data: &Value, now: u64) -> Result<(DeviceSignIn, DeviceSecret), String> { + let device_code = required(data, "device_code")?; + let user_code = required(data, "user_code")?; + let verification_uri = required(data, "verification_uri")?; + let url = url::Url::parse(&verification_uri).map_err(|_| "invalid Microsoft verification URL")?; + if url.scheme() != "https" + || !matches!(url.host_str(), Some("microsoft.com" | "www.microsoft.com" | "login.microsoftonline.com")) + || !url.username().is_empty() + || url.password().is_some() + || url.port().is_some_and(|port| port != 443) + || url.fragment().is_some() + { + return Err("Microsoft sign-in returned an unexpected verification URL".into()); + } + let expires_at = now.saturating_add( + data.get("expires_in").and_then(Value::as_u64).filter(|seconds| *seconds > 0 && *seconds <= 3600).ok_or("invalid sign-in expiry")?, + ); + let interval = data.get("interval").and_then(Value::as_u64).unwrap_or(5).clamp(5, 60); + let flow_id = format!("flow-{}", hash(device_code.as_bytes())); + Ok(( + DeviceSignIn { flow_id, user_code, verification_uri, expires_at, interval_seconds: interval }, + DeviceSecret { binding: binding(source), device_code, expires_at, next_poll: now.saturating_add(interval), interval }, + )) +} + +pub fn outlook_poll_sign_in(store: &CredentialStore, source: &OutlookSource, sign_in: &DeviceSignIn) -> Result { + guarded(|| { + validate(source)?; + let bytes = credentials::get(store, &sign_in.flow_id)?.ok_or("this sign-in is missing or already finished; begin a new sign-in")?; + let mut secret: DeviceSecret = serde_json::from_slice(&bytes).map_err(|_| "invalid sign-in state")?; + let network = Network::new()?; + let control = Control::new(&|| false); + match poll_device(&network, source, &mut secret, crate::hot_folder::now(), &control)? { + DevicePoll::Authorized(token) => { + store_token(store, source, &token)?; + credentials::forget_credentials(store, &sign_in.flow_id)?; + Ok(SignInStatus::Authorized) + } + DevicePoll::Finished(status) => { + credentials::forget_credentials(store, &sign_in.flow_id)?; + Ok(status) + } + DevicePoll::Pending(seconds) => { + credentials::put(store, &sign_in.flow_id, &Zeroizing::new(serde_json::to_vec(&secret).map_err(|_| "cannot encode sign-in state")?))?; + Ok(SignInStatus::Pending { retry_after_seconds: seconds }) + } + } + }) +} + +enum DevicePoll { + Pending(u64), + Authorized(OAuthSecret), + Finished(SignInStatus), +} +fn poll_device(network: &dyn Http, source: &OutlookSource, secret: &mut DeviceSecret, now: u64, control: &Control<'_>) -> Result { + control.check()?; + if secret.binding != binding(source) { + return Err("sign-in belongs to another Outlook application or permission scope".into()); + } + if now >= secret.expires_at { + return Ok(DevicePoll::Finished(SignInStatus::Expired)); + } + if now < secret.next_poll { + return Ok(DevicePoll::Pending(secret.next_poll - now)); + } + let body = form(&[ + ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), + ("client_id", &source.client_id), + ("device_code", &secret.device_code), + ]); + let response = network.request("POST", &auth_url(source, "token"), None, Some(&body), true, MAX_JSON, control)?; + let data = response.json()?; + if response.status == 200 { + let mut token = parse_token(source, &data, None, now)?; + token.account_id = account_id(network, &token.access_token, control)?; + return Ok(DevicePoll::Authorized(token)); + } + match data.get("error").and_then(Value::as_str).unwrap_or_default() { + "authorization_pending" => {} + "slow_down" => secret.interval = secret.interval.saturating_add(5).min(300), + "authorization_declined" | "access_denied" => return Ok(DevicePoll::Finished(SignInStatus::Declined)), + "expired_token" => return Ok(DevicePoll::Finished(SignInStatus::Expired)), + _ => { + response.success()?; + return Err("Microsoft sign-in could not be completed; begin a new sign-in".into()); + } + } + secret.next_poll = now.saturating_add(secret.interval); + Ok(DevicePoll::Pending(secret.interval)) +} + +fn required(value: &Value, key: &str) -> Result { + value + .get(key) + .and_then(Value::as_str) + .filter(|value| !value.is_empty() && value.len() <= 32 * 1024) + .map(str::to_owned) + .ok_or_else(|| format!("Microsoft response is missing {key}")) +} +fn parse_token(source: &OutlookSource, data: &Value, old: Option<&OAuthSecret>, now: u64) -> Result { + if data.get("token_type").and_then(Value::as_str).is_none_or(|value| !value.eq_ignore_ascii_case("bearer")) { + return Err("Microsoft returned an unsupported token type".into()); + } + let refresh_token = data + .get("refresh_token") + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| old.map(|old| old.refresh_token.clone())) + .ok_or("Microsoft did not grant offline access; sign in with offline_access")?; + let scopes = data.get("scope").and_then(Value::as_str).unwrap_or_default().to_owned(); + let write = !matches!(source.action, SourceAction::Keep); + let has_read_write = scopes + .split_whitespace() + .any(|scope| scope.eq_ignore_ascii_case("Mail.ReadWrite") || scope.eq_ignore_ascii_case("https://graph.microsoft.com/Mail.ReadWrite")); + let has_read = scopes + .split_whitespace() + .any(|scope| scope.eq_ignore_ascii_case("Mail.Read") || scope.eq_ignore_ascii_case("https://graph.microsoft.com/Mail.Read")); + if !has_read_write && (write || !has_read) { + return Err("Microsoft did not grant the required mail permission; check app consent and sign in again".into()); + } + let expires = + data.get("expires_in").and_then(Value::as_u64).filter(|value| *value > 0 && *value <= 86400).ok_or("invalid Microsoft token expiry")?; + Ok(OAuthSecret { + binding: binding(source), + access_token: required(data, "access_token")?, + refresh_token, + expires_at: now.saturating_add(expires), + account_id: old.map(|value| value.account_id.clone()).unwrap_or_default(), + scope: scopes, + }) +} +fn store_token(store: &CredentialStore, source: &OutlookSource, token: &OAuthSecret) -> Result<(), String> { + let bytes = Zeroizing::new(serde_json::to_vec(token).map_err(|_| "cannot encode Outlook credentials")?); + credentials::put(store, &source.credential_id, &bytes) +} +fn account_id(network: &dyn Http, token: &str, control: &Control<'_>) -> Result { + let response = network.request("GET", &format!("{GRAPH}/me?$select=id"), Some(token), None, false, MAX_JSON, control)?; + response.success()?; + required(&response.json()?, "id") +} + +fn refresh_token(network: &dyn Http, source: &OutlookSource, old: &OAuthSecret, control: &Control<'_>) -> Result { + let body = + form(&[("grant_type", "refresh_token"), ("client_id", &source.client_id), ("refresh_token", &old.refresh_token), ("scope", scope(source))]); + let response = network.request("POST", &auth_url(source, "token"), None, Some(&body), true, MAX_JSON, control)?; + if response.status != 200 { + return Err("Outlook refresh was rejected or unavailable; sign in again if retry does not recover".into()); + } + let mut token = parse_token(source, &response.json()?, Some(old), crate::hot_folder::now())?; + let account = account_id(network, &token.access_token, control)?; + if account != old.account_id { + return Err("Outlook refresh resolved to another account; original retained".into()); + } + token.account_id = account; + Ok(token) +} + +pub(super) struct Client<'a> { + store: &'a CredentialStore, + source: &'a OutlookSource, + network: Box, + token: OAuthSecret, +} +impl<'a> Client<'a> { + pub(super) fn connect(store: &'a CredentialStore, source: &'a OutlookSource) -> Result { + validate(source)?; + let bytes = credentials::get(store, &source.credential_id)?.ok_or("Outlook is not connected; start sign-in explicitly")?; + let token: OAuthSecret = serde_json::from_slice(&bytes).map_err(|_| "credential ID does not contain Outlook credentials")?; + if token.binding != binding(source) { + return Err("Outlook application or permissions changed; sign in again".into()); + } + Ok(Self { store, source, network: Box::new(Network::new()?), token }) + } + fn refresh(&mut self, control: &Control<'_>) -> Result<(), String> { + let token = refresh_token(self.network.as_ref(), self.source, &self.token, control)?; + store_token(self.store, self.source, &token)?; + self.token = token; + Ok(()) + } + fn request(&mut self, method: &str, url: &str, body: Option<&str>, limit: u64, control: &Control<'_>) -> Result { + graph_url_ok(url)?; + if self.token.expires_at <= crate::hot_folder::now().saturating_add(60) { + self.refresh(control)?; + } + let response = self.network.request(method, url, Some(&self.token.access_token), body, false, limit, control)?; + // Retry only reads: an uncertain successful POST/DELETE must be reconciled first. + if response.status == 401 && method == "GET" { + self.refresh(control)?; + return self.network.request(method, url, Some(&self.token.access_token), body, false, limit, control); + } + Ok(response) + } + fn collection(&mut self, url: String, control: &Control<'_>) -> Result, String> { + let mut next = Some(url); + let mut seen = std::collections::BTreeSet::new(); + let mut output = Vec::new(); + while let Some(url) = next.take() { + control.check()?; + if seen.len() >= 200 || !seen.insert(url.clone()) { + return Err("Outlook pagination exceeded its limit or repeated a page".into()); + } + let response = self.request("GET", &url, None, MAX_JSON, control)?; + response.success()?; + let data = response.json()?; + let values = data.get("value").and_then(Value::as_array).ok_or("Outlook collection response has no value array")?; + if output.len().saturating_add(values.len()) > MAX_ITEMS { + return Err("Outlook source exceeds 10000 items; choose a smaller mail folder".into()); + } + output.extend(values.iter().cloned()); + next = data.get("@odata.nextLink").and_then(Value::as_str).map(str::to_owned); + } + Ok(output) + } + fn attachments(&mut self, message: &str, control: &Control<'_>) -> Result, String> { + let url = format!("{}?$select=id,name,size,isInline,lastModifiedDateTime&$top=100", graph_url(&["me", "messages", message, "attachments"])?); + let values = self.collection(url, control)?; + let mut result = Vec::new(); + for value in values { + if value.get("@odata.type").and_then(Value::as_str) != Some("#microsoft.graph.fileAttachment") + || value.get("isInline").and_then(Value::as_bool) == Some(true) + { + continue; + } + let name = required(&value, "name")?; + let relative = PathBuf::from(hash(message.as_bytes()).get(..16).ok_or("invalid message ID hash")?).join(safe_filename(&name)); + if !supported(&relative) { + continue; + } + let id = required(&value, "id")?; + let modified = required(&value, "lastModifiedDateTime")?; + let size = value + .get("size") + .and_then(Value::as_u64) + .filter(|size| *size > 0 && *size <= MAX_FILE) + .ok_or("Outlook attachment is empty or exceeds 512 MiB")?; + result.push(RemoteFile { + key: id, + relative, + revision: format!("{}:{modified}", self.token.account_id), + size, + group: Some(message.into()), + }); + } + result.sort_by(|left, right| left.key.cmp(&right.key)); + if result.len() > MAX_BATCH || result.iter().fold(0_u64, |sum, file| sum.saturating_add(file.size)) > MAX_FILE { + return Err("one Outlook message exceeds the 128-attachment / 512 MiB batch limit; retain the mail and split its attachments".into()); + } + Ok(result) + } + fn download(&mut self, file: &RemoteFile, control: &Control<'_>) -> Result { + let message = file.group.as_deref().ok_or("Outlook receipt is missing its message ID")?; + let url = graph_url(&["me", "messages", message, "attachments", &file.key, "$value"])?; + let response = self.request("GET", &url, None, file.size, control)?; + response.success()?; + if response.body.len() as u64 != file.size { + return Err("Outlook attachment changed size during download".into()); + } + Ok(response) + } +} + +pub fn outlook_folders( + store: &CredentialStore, + source: &OutlookSource, + parent: Option<&str>, + should_stop: &dyn Fn() -> bool, +) -> Result, String> { + guarded(|| { + let mut client = Client::connect(store, source)?; + let control = Control::new(should_stop); + let url = + if let Some(parent) = parent { graph_url(&["me", "mailFolders", parent, "childFolders"])? } else { graph_url(&["me", "mailFolders"])? }; + let values = client.collection(format!("{url}?$select=id,displayName,childFolderCount&$top=100"), &control)?; + values + .iter() + .map(|value| { + Ok(MailFolder { + id: required(value, "id")?, + name: required(value, "displayName")?, + child_count: value.get("childFolderCount").and_then(Value::as_u64).unwrap_or(0), + }) + }) + .collect() + }) +} + +impl Backend for Client<'_> { + fn list(&mut self, control: &Control<'_>) -> Result, String> { + let url = format!( + "{}?$select=id,hasAttachments&$filter=hasAttachments%20eq%20true&$top=100", + graph_url(&["me", "mailFolders", &self.source.folder_id, "messages"])? + ); + let messages = self.collection(url, control)?; + let mut files = Vec::new(); + for message in messages { + files.extend(self.attachments(&required(&message, "id")?, control)?); + if files.len() > MAX_ITEMS { + return Err("Outlook source exceeds 10000 attachments; choose a smaller folder".into()); + } + } + Ok(files) + } + fn read(&mut self, file: &RemoteFile, writer: &mut dyn Write, control: &Control<'_>) -> Result<(), String> { + let message = file.group.as_deref().ok_or("Outlook receipt is missing its message ID")?; + let current = self.attachments(message, control)?; + if !current.iter().any(|current| current.key == file.key && current.revision == file.revision && current.size == file.size) { + return Err("Outlook attachment changed before download; retry after it settles".into()); + } + let response = self.download(file, control)?; + writer.write_all(&response.body).map_err(io_error) + } + fn finish(&mut self, records: &[Record], action: &SourceAction, control: &Control<'_>) -> Result<(), String> { + let first = records.first().ok_or("empty Outlook acknowledgement")?; + let message = first.remote.group.as_deref().ok_or("missing Outlook message ID")?; + let url = format!("{}?$select=id,parentFolderId", graph_url(&["me", "messages", message])?); + let response = self.request("GET", &url, None, MAX_JSON, control)?; + if response.status == 404 && matches!(action, SourceAction::Delete) { + return Ok(()); + } + response.success()?; + let message_data = response.json()?; + let destination = match action { + SourceAction::Move { destination } => Some(destination.as_str()), + SourceAction::Delete => Some("deleteditems"), + SourceAction::Keep => None, + }; + if let Some(destination) = destination { + let folder_url = format!("{}?$select=id", graph_url(&["me", "mailFolders", destination])?); + let destination_response = self.request("GET", &folder_url, None, MAX_JSON, control)?; + destination_response.success()?; + if required(&message_data, "parentFolderId")? == required(&destination_response.json()?, "id")? { + return Ok(()); + } + } + let watched = format!("{}?$select=id", graph_url(&["me", "mailFolders", &self.source.folder_id])?); + let watched = self.request("GET", &watched, None, MAX_JSON, control)?; + watched.success()?; + if required(&message_data, "parentFolderId")? != required(&watched.json()?, "id")? { + return Err("Outlook original left the watched folder; it was not moved or deleted".into()); + } + let current = self.attachments(message, control)?; + if current.len() != records.len() + || !current.iter().all(|file| { + records.iter().any(|record| { + record.ready && record.remote.key == file.key && record.remote.revision == file.revision && record.remote.size == file.size + }) + }) + { + return Err("Outlook message has unprocessed or changed attachments; the whole message was retained".into()); + } + for record in records { + if hash(&self.download(&record.remote, control)?.body) != record.source.sha256 { + return Err("Outlook attachment content changed; the whole message was retained".into()); + } + } + match action { + SourceAction::Keep => Ok(()), + SourceAction::Delete | SourceAction::Move { .. } => { + // "Delete" is a recoverable move to Deleted Items, not permanent deletion; + // an interrupted retry cannot delete the same mail a second time. + let destination = destination.ok_or("missing Outlook archive destination")?; + let body = json!({ "destinationId": destination }).to_string(); + self.request("POST", &graph_url(&["me", "messages", message, "move"])?, Some(&body), MAX_JSON, control)?.success() + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::collections::VecDeque; + use std::sync::{Arc, Mutex}; + struct FakeHttp { + replies: RefCell>, + calls: Arc>>, + } + impl Http for FakeHttp { + fn request( + &self, + method: &str, + url: &str, + _: Option<&str>, + body: Option<&str>, + _: bool, + _: u64, + control: &Control<'_>, + ) -> Result { + control.check()?; + self.calls.lock().unwrap().push((format!("{method} {url}"), body.unwrap_or_default().into())); + Ok(self.replies.borrow_mut().pop_front().expect("unexpected HTTP request")) + } + } + fn reply(value: Value) -> Response { + Response { status: 200, body: Zeroizing::new(serde_json::to_vec(&value).unwrap()), retry_after: 0 } + } + fn attachments() -> Value { + json!({"value":[ + {"@odata.type":"#microsoft.graph.fileAttachment","id":"a","name":"one.pdf","size":3,"lastModifiedDateTime":"2026-10-11T00:00:00Z"}, + {"@odata.type":"#microsoft.graph.fileAttachment","id":"b","name":"two.png","size":3,"lastModifiedDateTime":"2026-10-11T00:00:00Z"}, + {"@odata.type":"#microsoft.graph.referenceAttachment","id":"c","name":"cloud.pdf","size":3}, + {"@odata.type":"#microsoft.graph.fileAttachment","id":"d","name":"signature.png","size":3,"isInline":true} + ]}) + } + fn client<'a>(source: &'a OutlookSource, replies: Vec, calls: Arc>>) -> Client<'a> { + Client { + source, + store: &CredentialStore::OsVault, + network: Box::new(FakeHttp { replies: RefCell::new(replies.into()), calls }), + token: OAuthSecret { + binding: binding(source), + access_token: "test-only-access".into(), + refresh_token: "test-only-refresh".into(), + account_id: "account".into(), + scope: "Mail.ReadWrite".into(), + expires_at: crate::hot_folder::now().saturating_add(3600), + }, + } + } + fn records() -> Vec { + ["a", "b"] + .iter() + .map(|id| Record { + source: StagedSource { + source_id: (*id).into(), + relative: format!("{id}.pdf").into(), + staged_relative: format!("stage-{id}.pdf").into(), + sha256: hash(b"abc"), + }, + remote: RemoteFile { + key: (*id).into(), + relative: format!("{id}.pdf").into(), + revision: "account:2026-10-11T00:00:00Z".into(), + size: 3, + group: Some("immutable-message".into()), + }, + ready: true, + acknowledged: false, + }) + .collect() + } + fn before_finish() -> Vec { + vec![reply(json!({"parentFolderId":"inbox-id"})), reply(json!({"id":"deleted-id"})), reply(json!({"id":"inbox-id"})), reply(attachments())] + } + #[test] + fn outlook_keeps_whole_message_until_every_supported_attachment_is_ready() { + let source = OutlookSource { action: SourceAction::Delete, ..source() }; + let calls = Arc::new(Mutex::new(Vec::new())); + let mut client = client(&source, before_finish(), calls.clone()); + let mut records = records(); + records.pop(); + assert!(client.finish(&records, &SourceAction::Delete, &Control::new(&|| false)).is_err()); + assert!(calls.lock().unwrap().iter().all(|(method, _)| method.starts_with("GET "))); + } + #[test] + fn outlook_checks_each_attachment_then_moves_to_deleted_items_and_reconciles_retry() { + let source = OutlookSource { action: SourceAction::Delete, ..source() }; + let calls = Arc::new(Mutex::new(Vec::new())); + let mut responses = before_finish(); + for _ in 0..2 { + responses.push(Response { status: 200, body: Zeroizing::new(b"abc".to_vec()), retry_after: 0 }); + } + responses.push(reply(json!({"id":"immutable-message","parentFolderId":"deleted-id"}))); + responses.push(reply(json!({"id":"immutable-message","parentFolderId":"deleted-id"}))); + responses.push(reply(json!({"id":"deleted-id"}))); + let mut client = client(&source, responses, calls.clone()); + client.finish(&records(), &SourceAction::Delete, &Control::new(&|| false)).unwrap(); + client.finish(&records(), &SourceAction::Delete, &Control::new(&|| false)).unwrap(); + let calls = calls.lock().unwrap(); + assert_eq!(calls.iter().filter(|(method, _)| method.starts_with("POST ")).count(), 1); + assert_eq!(calls.iter().find(|(method, _)| method.starts_with("POST ")).unwrap().1, "{\"destinationId\":\"deleteditems\"}"); + assert!(calls.iter().all(|(method, _)| !method.starts_with("DELETE "))); + } + #[test] + fn outlook_pagination_escapes_and_http_failures_never_echo_response_secrets() { + let source = source(); + let mut client = + client(&source, vec![reply(json!({"value":[],"@odata.nextLink":"https://attacker.test/token"}))], Arc::new(Mutex::new(Vec::new()))); + assert!(client.collection(format!("{GRAPH}/me/mailFolders"), &Control::new(&|| false)).is_err()); + let response = Response { status: 403, body: Zeroizing::new(b"response containing secret NEVER-ECHO".to_vec()), retry_after: 0 }; + assert!(!response.success().unwrap_err().contains("NEVER-ECHO")); + } + fn source() -> OutlookSource { + OutlookSource { client_id: "01234567-89ab-cdef-0123-456789abcdef".into(), credential_id: "test".into(), ..Default::default() } + } + #[test] + fn device_flow_obeys_poll_interval_slow_down_consent_and_refresh_account_binding() { + let source = source(); + let control = Control::new(&|| false); + let mut secret = DeviceSecret { binding: binding(&source), device_code: "secret+=&".into(), expires_at: 1000, next_poll: 105, interval: 5 }; + let calls = Arc::new(Mutex::new(Vec::new())); + let error = |name| Response { status: 400, body: Zeroizing::new(serde_json::to_vec(&json!({"error":name})).unwrap()), retry_after: 0 }; + let token = || { + reply( + json!({"token_type":"Bearer","access_token":"test-access","refresh_token":"test-refresh","scope":"Mail.Read User.Read","expires_in":3600}), + ) + }; + let network = FakeHttp { + calls: calls.clone(), + replies: RefCell::new(vec![error("authorization_pending"), error("slow_down"), token(), reply(json!({"id":"account"}))].into()), + }; + assert!(matches!(poll_device(&network, &source, &mut secret, 101, &control).unwrap(), DevicePoll::Pending(4))); + assert!(calls.lock().unwrap().is_empty()); + assert!(matches!(poll_device(&network, &source, &mut secret, 105, &control).unwrap(), DevicePoll::Pending(5))); + assert_eq!(secret.next_poll, 110); + assert!(matches!(poll_device(&network, &source, &mut secret, 110, &control).unwrap(), DevicePoll::Pending(10))); + assert_eq!(secret.next_poll, 120); + assert!(matches!(poll_device(&network, &source, &mut secret, 119, &control).unwrap(), DevicePoll::Pending(1))); + let DevicePoll::Authorized(authorized) = poll_device(&network, &source, &mut secret, 120, &control).unwrap() else { + panic!("expected authorization") + }; + assert_eq!(authorized.account_id, "account"); + let log = calls.lock().unwrap(); + assert_eq!(log.len(), 4); + assert!(log[0].0.starts_with("POST https://login.microsoftonline.com/common/oauth2/v2.0/token")); + assert!(log[0].1.contains("device_code=secret%2B%3D%26")); + drop(log); + let network = FakeHttp { + calls: calls.clone(), + replies: RefCell::new(vec![token(), reply(json!({"id":"account"})), token(), reply(json!({"id":"other-account"}))].into()), + }; + assert_eq!(refresh_token(&network, &source, &authorized, &control).unwrap().account_id, "account"); + assert!(refresh_token(&network, &source, &authorized, &control).is_err()); + assert!(matches!(poll_device(&network, &source, &mut secret, 1000, &control).unwrap(), DevicePoll::Finished(SignInStatus::Expired))); + let network = FakeHttp { calls, replies: RefCell::new(vec![error("access_denied")].into()) }; + assert!(matches!(poll_device(&network, &source, &mut secret, 130, &control).unwrap(), DevicePoll::Finished(SignInStatus::Declined))); + } + #[test] + fn device_flow_never_serializes_the_device_secret_and_limits_urls() { + let data = json!({ "device_code": "DO-NOT-EXPOSE", "user_code": "ABCD-EFGH", "verification_uri": "https://microsoft.com/devicelogin", "expires_in":900, "interval":5 }); + let (public, secret) = parse_device(&source(), &data, 100).unwrap(); + assert_eq!(public.expires_at, 1000); + assert_eq!(secret.next_poll, 105); + assert!(!serde_json::to_string(&public).unwrap().contains("DO-NOT-EXPOSE")); + let mut wrong = data; + wrong["verification_uri"] = json!("https://attacker.example/login"); + assert!(parse_device(&source(), &wrong, 100).is_err()); + } + #[test] + fn pagination_keeps_bearer_tokens_on_the_expected_https_mailbox_host() { + assert!(graph_url_ok("https://graph.microsoft.com/v1.0/me/mailFolders/inbox/messages?$skiptoken=a").is_ok()); + for url in [ + "http://graph.microsoft.com/v1.0/me/messages", + "https://graph.microsoft.com.attacker.test/v1.0/me/messages", + "https://user@graph.microsoft.com/v1.0/me/messages", + "https://graph.microsoft.com:123/v1.0/me/messages", + "https://graph.microsoft.com/v1.0/users/other/messages", + ] { + assert!(graph_url_ok(url).is_err()); + } + let url = graph_url(&["me", "messages", "a/b+="]).unwrap(); + assert!(url.contains("a%2Fb+=")); + } + #[test] + fn tokens_require_mail_consent_and_keep_refresh_on_rotation_omission() { + let source = source(); + let data = json!({ "token_type":"Bearer", "access_token":"opaque-access", "refresh_token":"opaque-refresh", "expires_in":3600, "scope":"User.Read Mail.Read" }); + let old = parse_token(&source, &data, None, 100).unwrap(); + let mut refreshed = data.clone(); + refreshed.as_object_mut().unwrap().remove("refresh_token"); + assert_eq!(parse_token(&source, &refreshed, Some(&old), 200).unwrap().refresh_token, "opaque-refresh"); + let write = OutlookSource { action: SourceAction::Delete, ..source }; + assert!(parse_token(&write, &data, None, 100).is_err()); + } +} diff --git a/crates/engine/src/hot_folder_sources/project.rs b/crates/engine/src/hot_folder_sources/project.rs new file mode 100644 index 0000000000..e66dffb6e4 --- /dev/null +++ b/crates/engine/src/hot_folder_sources/project.rs @@ -0,0 +1,211 @@ +//! PDFCraft's open project manifest. This is not an ABBYY proprietary OCR project. + +use super::*; + +pub(super) fn local_paths(source: &ProjectSource, root: Option<&Path>) -> Result, String> { + let (base, manifest) = manifest(source, root)?; + manifest.files.iter().map(|relative| confined(&base.join(relative), root)).collect() +} + +/// An unprocessed missing project entry is an error, never a silently truncated +/// grouped output. Completed move/delete entries stay in the immutable manifest. +pub(super) fn validate_and_order(source: &ProjectSource, root: Option<&Path>, journal: &Journal, report: &mut StageReport) -> Result<(), String> { + let (base, manifest) = manifest(source, root)?; + for relative in &manifest.files { + if fs::symlink_metadata(base.join(relative)).is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound) + && !journal.records.values().any(|record| record.remote.relative == *relative) + { + return Err("project contains a missing, unprocessed source; restore it before processing the project".into()); + } + } + let positions = manifest.files.iter().enumerate().map(|(index, path)| (path, index)).collect::>(); + if report.items.iter().any(|item| !positions.contains_key(&item.relative)) { + return Err("project manifest changed while staged inputs are pending; restore its entries or use a fresh task".into()); + } + report.items.sort_by_key(|item| positions.get(&item.relative).copied().unwrap_or(usize::MAX)); + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectManifest { + /// Must be "pdfcraft-hot-folder-project". + pub format: String, + pub version: u32, + /// Ordered PDF/PNG/JPEG/TIFF paths relative to this manifest, without parent traversal. + pub files: Vec, +} + +impl Default for ProjectManifest { + fn default() -> Self { + Self { format: "pdfcraft-hot-folder-project".into(), version: 1, files: Vec::new() } + } +} + +pub fn write_project_manifest(path: &Path, manifest: &ProjectManifest, root: Option<&Path>) -> Result<(), String> { + manifest_check(manifest)?; + let parent = confined(path.parent().ok_or("project manifest needs a parent directory")?, root)?; + let filename = path.file_name().ok_or("project manifest needs a filename")?; + for file in &manifest.files { + confined(&parent.join(file), root)?; + } + let bytes = serde_json::to_vec_pretty(manifest).map_err(io_error)?; + if bytes.len() as u64 > MAX_JSON { + return Err("project manifest exceeds 8 MiB".into()); + } + let mut temporary = tempfile::Builder::new().prefix(".pdfcraft-project-").tempfile_in(&parent).map_err(io_error)?; + temporary.write_all(&bytes).and_then(|_| temporary.as_file().sync_all()).map_err(io_error)?; + temporary.persist_noclobber(parent.join(filename)).map_err(|_| "project manifest already exists or could not be saved; no file was replaced")?; + sync_directory(&parent) +} + +fn manifest_check(manifest: &ProjectManifest) -> Result<(), String> { + if manifest.format != "pdfcraft-hot-folder-project" || manifest.version != 1 { + return Err("use PDFCraft's open version-1 project manifest; proprietary ABBYY OCR projects are not supported".into()); + } + if manifest.files.is_empty() || manifest.files.len() > MAX_ITEMS { + return Err("project must contain 1..10000 PDF, PNG, JPEG or TIFF files".into()); + } + let mut unique = std::collections::BTreeSet::new(); + for path in &manifest.files { + if !relative_ok(path) || !supported(path) || !unique.insert(path) { + return Err("project paths must be unique relative PDF/image paths without traversal".into()); + } + } + Ok(()) +} + +fn manifest(source: &ProjectSource, root: Option<&Path>) -> Result<(PathBuf, ProjectManifest), String> { + let path = confined(&source.manifest, root)?; + let bytes = read_bounded(&path, MAX_JSON)?; + let parsed: ProjectManifest = + serde_json::from_slice(&bytes).map_err(|_| "invalid PDFCraft project manifest; ABBYY OCR projects are not supported".to_owned())?; + manifest_check(&parsed)?; + Ok((path.parent().ok_or("project manifest needs a parent directory")?.to_owned(), parsed)) +} + +pub(super) fn validate(source: &ProjectSource, input: &Path, root: Option<&Path>) -> Result<(), String> { + let (base, parsed) = manifest(source, root)?; + for relative in parsed.files { + let path = confined(&base.join(relative), root)?; + if path.starts_with(input) { + return Err("project originals must be outside their staging inbox".into()); + } + } + if let SourceAction::Move { destination } = &source.action { + let archive = confined(Path::new(destination), root)?; + if !archive.is_dir() || archive.starts_with(input) || input.starts_with(&archive) { + return Err("project archive must be an existing folder disjoint from staging".into()); + } + } + Ok(()) +} + +pub(super) struct Client<'a> { + source: &'a ProjectSource, + root: Option<&'a Path>, + base: PathBuf, + manifest: ProjectManifest, +} +impl<'a> Client<'a> { + pub(super) fn new(source: &'a ProjectSource, root: Option<&'a Path>) -> Result { + let (base, manifest) = manifest(source, root)?; + Ok(Self { source, root, base, manifest }) + } + fn path(&self, file: &RemoteFile) -> Result { + if !self.manifest.files.contains(&file.relative) { + return Err("project receipt is not listed in the manifest".into()); + } + let path = confined(&self.base.join(&file.relative), self.root)?; + if path.to_string_lossy() != file.key { + return Err("project receipt resolved to another source path".into()); + } + Ok(path) + } +} +impl Backend for Client<'_> { + fn list(&mut self, control: &Control<'_>) -> Result, String> { + let mut result = Vec::new(); + for relative in &self.manifest.files { + control.check()?; + let candidate = self.base.join(relative); + // A previous acknowledged move/delete leaves the immutable manifest intact. + if fs::symlink_metadata(&candidate).is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound) { + continue; + } + let path = confined(&candidate, self.root)?; + let revision = file_hash(&path, control)?; + result.push(RemoteFile { + key: path.to_string_lossy().into_owned(), + relative: relative.clone(), + revision, + size: fs::metadata(&path).map_err(io_error)?.len(), + group: None, + }); + } + Ok(result) + } + fn read(&mut self, file: &RemoteFile, writer: &mut dyn Write, control: &Control<'_>) -> Result<(), String> { + let path = self.path(file)?; + if file_hash(&path, control)? != file.revision { + return Err("project source changed before staging".into()); + } + let count = copy_limited(&mut File::open(&path).map_err(io_error)?, writer, file.size, control)?; + if count != file.size || file_hash(&path, control)? != file.revision { + return Err("project source changed during staging".into()); + } + Ok(()) + } + fn finish(&mut self, files: &[Record], action: &SourceAction, control: &Control<'_>) -> Result<(), String> { + let _ = &self.source; + for record in files { + control.check()?; + let candidate = self.base.join(&record.remote.relative); + let exists = !fs::symlink_metadata(&candidate).is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound); + let path = if exists { Some(self.path(&record.remote)?) } else { None }; + if let Some(path) = &path + && file_hash(path, control)? != record.source.sha256 + { + return Err("project original changed after processing; original retained".into()); + } + match action { + SourceAction::Keep => {} + SourceAction::Delete => { + if let Some(path) = path { + fs::remove_file(&path).map_err(io_error)?; + sync_directory(path.parent().ok_or("project original needs a parent directory")?)?; + } + } + SourceAction::Move { destination } => { + let archive = confined(Path::new(destination), self.root)?; + let target = archive.join(&record.source.staged_relative); + if target.exists() { + if file_hash(&target, control)? != record.source.sha256 { + return Err("project archive filename is occupied; original retained".into()); + } + } else { + let source = path.as_ref().ok_or("project original and archive are both missing")?; + let mut temporary = tempfile::Builder::new().prefix(".pdfcraft-project-").tempfile_in(&archive).map_err(io_error)?; + copy_limited(&mut File::open(source).map_err(io_error)?, &mut temporary, MAX_FILE, control)?; + temporary.as_file().sync_all().map_err(io_error)?; + if file_hash(temporary.path(), control)? != record.source.sha256 { + return Err("project original changed during archive copy; original retained".into()); + } + temporary.persist_noclobber(&target).map_err(|_| "project archive filename became occupied; original retained")?; + } + // Also confirm an existing matching archive on recovery; a previous + // process may have stopped between its rename and directory sync. + sync_directory(&archive)?; + if let Some(path) = path { + if file_hash(&path, control)? != record.source.sha256 { + return Err("project original changed before removal; original retained".into()); + } + fs::remove_file(&path).map_err(io_error)?; + sync_directory(path.parent().ok_or("project original needs a parent directory")?)?; + } + } + } + } + Ok(()) + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 278da22eb5..b8efe600ae 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -18,6 +18,18 @@ pub mod commands; pub mod compare; pub mod dates; pub mod export; +#[cfg(not(target_arch = "wasm32"))] +pub mod hot_folder; +#[cfg(not(target_arch = "wasm32"))] +pub mod hot_folder_analysis; +#[cfg(not(target_arch = "wasm32"))] +pub mod hot_folder_output; +#[cfg(not(target_arch = "wasm32"))] +pub mod hot_folder_preprocess; +#[cfg(not(target_arch = "wasm32"))] +mod hot_folder_schedule; +#[cfg(not(target_arch = "wasm32"))] +pub mod hot_folder_sources; pub mod js; pub mod links; pub mod ocr; diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs index 4eaf0fc450..b7c9634ec1 100644 --- a/crates/engine/src/ocr.rs +++ b/crates/engine/src/ocr.rs @@ -16,6 +16,8 @@ use crate::{DocId, Edit, EditError, Session}; /// Recognize Text settings. #[derive(Clone, Debug, PartialEq)] +#[cfg_attr(not(target_arch = "wasm32"), derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(not(target_arch = "wasm32"), serde(default, deny_unknown_fields))] pub struct OcrSettings { /// Resolution pages are rendered at for recognition (Acrobat's "Downsample to" choices). pub dpi: f32, diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 5e78640a19..ecfffe98a2 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -7,8 +7,11 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod publications; mod zip; +pub use publications::{epub, xlsx}; + pub use zip::Zip; /// A paragraph of a page. diff --git a/crates/export/src/publications.rs b/crates/export/src/publications.rs new file mode 100644 index 0000000000..d0255e555c --- /dev/null +++ b/crates/export/src/publications.rs @@ -0,0 +1,444 @@ +//! SpreadsheetML and EPUB 3.3 writers. Content is reflowed, not a facsimile. +//! https://learn.microsoft.com/office/open-xml/spreadsheet/structure-of-a-spreadsheetml-document +//! https://www.w3.org/TR/epub-33/ + +use super::{Item, Page, Zip, esc, hex, items}; + +const REL: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; +const PKG_REL: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; +const SHEET: &str = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; +const XML: &str = ""; +const MAX_BYTES: usize = 256 * 1024 * 1024; + +/// Bounds the legacy ZIP32 writer before every addition, including generated XML. +struct Archive { + zip: Zip, + size: usize, + entries: usize, +} + +impl Archive { + fn new() -> Self { + Self { zip: Zip::default(), size: 0, entries: 0 } + } + fn add(&mut self, name: &str, data: &[u8], compress: bool) -> Result<(), String> { + self.size = self.size.saturating_add(data.len()).saturating_add(name.len() * 2 + 128); + self.entries += 1; + if self.size > MAX_BYTES || self.entries > 60_000 { + return Err("export exceeds the 256 MiB or 60000-file archive limit; export fewer pages".into()); + } + self.zip.add(name, data, compress); + Ok(()) + } +} + +fn validate(pages: &[Page], title: &str) -> Result<(), String> { + if pages.is_empty() || pages.len() > 10_000 || title.len() > 16_384 { + return Err("export needs 1..10000 pages and a title of at most 16384 bytes".into()); + } + let mut bytes = 0usize; + for page in pages { + if page.blocks.len() > 100_000 || page.images.len() > 10_000 { + return Err("page exceeds the export block/image limit".into()); + } + for block in &page.blocks { + if block.text.len() > 8 * 1024 * 1024 { + return Err("text block exceeds the 8 MiB export limit".into()); + } + bytes = bytes.saturating_add(block.text.len()); + } + for image in &page.images { + if !matches!(image.ext, "png" | "jpg") { + return Err("export supports PNG and JPEG images only".into()); + } + bytes = bytes.saturating_add(image.bytes.len()); + } + if bytes > MAX_BYTES / 8 { + return Err("export input exceeds 32 MiB; export fewer pages".into()); + } + } + Ok(()) +} + +fn relationships(body: &str) -> String { + format!("{XML}{body}") +} + +fn relationship(id: &str, kind: &str, target: &str) -> String { + format!("") +} + +fn column(mut index: usize) -> String { + let mut chars = Vec::new(); + loop { + chars.push(char::from(b'A' + (index % 26) as u8)); + if index < 26 { + break; + } + index = index / 26 - 1; + } + chars.into_iter().rev().collect() +} + +/// Escape SpreadsheetML's literal `_xHHHH_` sequences before XML escaping. +fn sheet_text(text: &str) -> String { + let mut out = String::new(); + for (offset, c) in text.char_indices() { + if c == '_' + && text + .get(offset..offset.saturating_add(7)) + .is_some_and(|s| s.starts_with("_x") && s.ends_with('_') && s.get(2..6).is_some_and(|hex| hex.bytes().all(|c| c.is_ascii_hexdigit()))) + { + out.push_str("_x005F_"); + } else { + out.push(c); + } + } + esc(&out) +} + +fn cell(text: &str, col: usize, row: usize, bold: bool, italic: bool, color: [f64; 3]) -> Result { + // Excel's limit is UTF-16 code units, not bytes or Unicode scalar values. + if text.encode_utf16().count() > 32_767 { + return Err("a spreadsheet cell exceeds Excel's 32767-character limit; use another output format".into()); + } + let mut properties = String::new(); + if bold { + properties.push_str(""); + } + if italic { + properties.push_str(""); + } + if let Some(color) = hex(color) { + properties.push_str(&format!("")); + } + // Always strings: OCR must not silently change identifiers, dates or formulas. + Ok(format!( + "{properties}{}", + column(col), + sheet_text(text) + )) +} + +/// One worksheet per PDF page. Detected tables become cells/merged ranges; other +/// paragraphs become rows. Images stay at their position in the reading order. +/// Text is never inferred to be a number or an executable spreadsheet formula. +pub fn xlsx(pages: &[Page], title: &str) -> Result, String> { + validate(pages, title)?; + let mut archive = Archive::new(); + let mut types = String::from( + "", + ); + let mut sheets = String::new(); + let mut workbook_rels = relationship("styles", "styles", "styles.xml"); + for (index, page) in pages.iter().enumerate() { + let number = index + 1; + sheets.push_str(&format!("")); + workbook_rels.push_str(&relationship(&format!("s{number}"), "worksheet", &format!("worksheets/sheet{number}.xml"))); + types.push_str(&format!("")); + let mut rows = String::new(); + let mut merges = Vec::new(); + let mut drawing = String::new(); + let mut drawing_rels = String::new(); + let mut row = 1usize; + let mut image_number = 0usize; + for item in items(std::slice::from_ref(page)) { + match item { + Item::Para(block, _) => { + rows.push_str(&format!("{}", cell(&block.text, 0, row, block.bold, block.italic, block.color)?)); + row += 1; + } + Item::Table(table) => { + for cells in table.rows { + rows.push_str(&format!("")); + let mut col = 0; + for c in cells { + rows.push_str(&cell(&c.text, col, row, c.bold, c.italic, c.color)?); + if c.span > 1 { + merges.push(format!("", column(col), column(col + c.span - 1))); + } + col += c.span; + } + rows.push_str(""); + row += 1; + } + } + Item::Img(image) => { + image_number += 1; + let name = format!("page{number}-image{image_number}.{}", image.ext); + archive.add(&format!("xl/media/{name}"), &image.bytes, false)?; + let width = finite_size(image.rect[2] - image.rect[0]); + let height = finite_size(image.rect[3] - image.rect[1]); + let scale = (480.0 / width).min(600.0 / height).min(1.0); + let cx = (width * scale * 12700.0).round() as u64; + let cy = (height * scale * 12700.0).round() as u64; + drawing.push_str(&format!("00{}0", row - 1)); + drawing_rels.push_str(&relationship(&format!("i{image_number}"), "image", &format!("../media/{name}"))); + row += (height * scale / 15.0).ceil() as usize + 1; + } + Item::PageBreak => {} + } + if row > 1_048_576 || rows.len() > MAX_BYTES / 2 { + return Err("worksheet exceeds Excel's row limit or the export size limit".into()); + } + } + let merges = + if merges.is_empty() { String::new() } else { format!("{}", merges.len(), merges.concat()) }; + let has_images = image_number > 0; + if has_images { + types.push_str(&format!( + "" + )); + archive.add(&format!("xl/drawings/drawing{number}.xml"), format!("{XML}{drawing}").as_bytes(), true)?; + archive.add(&format!("xl/drawings/_rels/drawing{number}.xml.rels"), relationships(&drawing_rels).as_bytes(), true)?; + archive.add( + &format!("xl/worksheets/_rels/sheet{number}.xml.rels"), + relationships(&relationship("drawing", "drawing", &format!("../drawings/drawing{number}.xml"))).as_bytes(), + true, + )?; + } + let drawing_ref = if has_images { "" } else { "" }; + archive.add(&format!("xl/worksheets/sheet{number}.xml"), format!("{XML}{rows}{merges}{drawing_ref}").as_bytes(), true)?; + } + archive.add("xl/styles.xml", format!("{XML}").as_bytes(), true)?; + archive.add( + "xl/workbook.xml", + format!("{XML}{sheets}").as_bytes(), + true, + )?; + archive.add("xl/_rels/workbook.xml.rels", relationships(&workbook_rels).as_bytes(), true)?; + archive.add("_rels/.rels", relationships(&relationship("workbook", "officeDocument", "xl/workbook.xml")).as_bytes(), true)?; + archive.add( + "[Content_Types].xml", + format!("{XML}{types}").as_bytes(), + true, + )?; + Ok(archive.zip.finish()) +} + +fn finite_size(value: f64) -> f64 { + if value.is_finite() { value.clamp(1.0, 14400.0) } else { 72.0 } +} + +fn rich(text: &str, bold: bool, italic: bool) -> String { + let mut text = esc(text); + if italic { + text = format!("{text}"); + } + if bold { + text = format!("{text}"); + } + text +} + +/// EPUB 3.3 with a navigation document, one reflowable XHTML chapter per page, +/// real tables and packaged images. `language` is a BCP-47 tag, or `und` if unknown. +/// Reproducible exports use the ZIP epoch for the required modification timestamp. +pub fn epub(pages: &[Page], title: &str, language: &str) -> Result, String> { + validate(pages, title)?; + if language.split('-').next().is_none_or(|primary| !(2..=8).contains(&primary.len()) || !primary.bytes().all(|c| c.is_ascii_alphabetic())) + || language.len() > 64 + || !language.split('-').all(|part| !part.is_empty() && part.len() <= 8 && part.bytes().all(|c| c.is_ascii_alphanumeric())) + { + return Err("EPUB language must be a BCP-47 language tag (use und when unknown)".into()); + } + let mut archive = Archive::new(); + archive.add("mimetype", b"application/epub+zip", false)?; + archive.add("META-INF/container.xml", format!("{XML}").as_bytes(), true)?; + let mut manifest = String::from(""); + let mut spine = String::new(); + let mut nav = String::new(); + let mut fingerprint = 14695981039346656037u64; + for byte in title.as_bytes() { + fingerprint = (fingerprint ^ u64::from(*byte)).wrapping_mul(1099511628211); + } + for (index, page) in pages.iter().enumerate() { + let number = index + 1; + let mut body = String::new(); + let mut image_number = 0; + for item in items(std::slice::from_ref(page)) { + match item { + Item::Para(block, level) => { + let tag = if level == 0 { "p".to_owned() } else { format!("h{level}") }; + body.push_str(&format!("<{tag}>{}", rich(&block.text, block.bold, block.italic))); + } + Item::Table(table) => { + body.push_str(""); + for row in table.rows { + body.push_str(""); + for cell in row { + body.push_str(&format!("", cell.span, rich(&cell.text, cell.bold, cell.italic))); + } + body.push_str(""); + } + body.push_str("
{}
"); + } + Item::Img(image) => { + image_number += 1; + let name = format!("images/page{number}-image{image_number}.{}", image.ext); + let mime = if image.ext == "jpg" { "image/jpeg" } else { "image/png" }; + archive.add(&format!("EPUB/{name}"), &image.bytes, false)?; + manifest.push_str(&format!("")); + body.push_str(&format!("

\"Image

")); + for byte in &image.bytes { + fingerprint = (fingerprint ^ u64::from(*byte)).wrapping_mul(1099511628211); + } + } + Item::PageBreak => {} + } + if body.len() > MAX_BYTES / 2 { + return Err("EPUB page exceeds the export size limit".into()); + } + } + for byte in body.as_bytes() { + fingerprint = (fingerprint ^ u64::from(*byte)).wrapping_mul(1099511628211); + } + archive.add(&format!("EPUB/page{number}.xhtml"), xhtml(&format!("{title} — {number}"), language, &body).as_bytes(), true)?; + manifest.push_str(&format!("")); + spine.push_str(&format!("")); + nav.push_str(&format!("
  • {number}
    • ")); + } + archive.add( + "EPUB/nav.xhtml", + xhtml(title, language, &format!("", esc(title))).as_bytes(), + true, + )?; + archive.add("EPUB/package.opf", format!("{XML}urn:pdfcraft:{fingerprint:016x}{}{language}1980-01-01T00:00:00Z{manifest}{spine}", esc(if title.trim().is_empty() { "PdfCraft document" } else { title })).as_bytes(), true)?; + Ok(archive.zip.finish()) +} + +fn xhtml(title: &str, language: &str, body: &str) -> String { + format!( + "{XML}{}{body}", + esc(title) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Block, Image}; + use std::{collections::BTreeMap, io::Read}; + + fn parts(bytes: &[u8]) -> BTreeMap> { + let mut parts = BTreeMap::new(); + let mut offset = 0; + while bytes[offset..].starts_with(b"PK\x03\x04") { + let word = |at: usize, len: usize| bytes[at..at + len].iter().rev().fold(0usize, |n, b| n * 256 + usize::from(*b)); + let method = word(offset + 8, 2); + let size = word(offset + 18, 4); + let name_len = word(offset + 26, 2); + let extra_len = word(offset + 28, 2); + let name = String::from_utf8(bytes[offset + 30..offset + 30 + name_len].to_vec()).unwrap(); + let start = offset + 30 + name_len + extra_len; + let body = &bytes[start..start + size]; + let data = if method == 0 { + body.to_vec() + } else { + let mut data = Vec::new(); + flate2::read::DeflateDecoder::new(body).read_to_end(&mut data).unwrap(); + data + }; + assert!(parts.insert(name, data).is_none()); + offset = start + size; + } + parts + } + + fn fixture() -> Page { + let mut page = Page { width: 600.0, height: 800.0, ..Page::default() }; + for (row, texts) in [["=1+1", "00123"], ["日本語", "français"], ["_x0041_", "<&>\" — ß"]].iter().enumerate() { + for (col, text) in texts.iter().enumerate() { + let x = 40.0 + col as f64 * 200.0; + let y = 600.0 - row as f64 * 30.0; + page.blocks.push(Block { + text: (*text).into(), + rect: [x, y, x + 100.0, y + 12.0], + size: 12.0, + bold: row == 0, + italic: false, + color: [0.0; 3], + }); + } + } + // Original synthetic 1x1 PNG. Binary data is generated by the fixture code, + // not a downloaded/committed document asset. + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + let chunk = |out: &mut Vec, kind: &[u8; 4], data: &[u8]| { + out.extend((data.len() as u32).to_be_bytes()); + out.extend(kind); + out.extend(data); + let mut crc = 0xffff_ffffu32; + for byte in kind.iter().chain(data) { + crc ^= u32::from(*byte); + for _ in 0..8 { + crc = (crc >> 1) ^ if crc & 1 != 0 { 0xedb8_8320 } else { 0 }; + } + } + out.extend((!crc).to_be_bytes()); + }; + chunk(&mut png, b"IHDR", &[0, 0, 0, 1, 0, 0, 0, 1, 8, 2, 0, 0, 0]); + let mut z = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + std::io::Write::write_all(&mut z, &[0, 30, 90, 170]).unwrap(); + chunk(&mut png, b"IDAT", &z.finish().unwrap()); + chunk(&mut png, b"IEND", &[]); + page.images.push(Image { ext: "png", bytes: png, rect: [40.0, 400.0, 140.0, 500.0] }); + page + } + + #[test] + fn spreadsheets_preserve_text_tables_and_images_without_formulas() { + let bytes = xlsx(&[fixture(), fixture()], "Report").unwrap(); + let parts = parts(&bytes); + let sheet = String::from_utf8(parts["xl/worksheets/sheet1.xml"].clone()).unwrap(); + for expected in + ["r=\"A1\" t=\"inlineStr\"", "r=\"B1\" t=\"inlineStr\"", "=1+1", "00123", "日本語", "français", "_x005F_x0041_", "<&>""] + { + assert!(sheet.contains(expected), "{expected}"); + } + assert!(!sheet.contains("")); + assert!(parts.contains_key("xl/worksheets/sheet2.xml")); + assert_eq!(parts["xl/media/page1-image1.png"], fixture().images[0].bytes); + assert!(String::from_utf8(parts["xl/drawings/drawing1.xml"].clone()).unwrap().contains("r:embed=\"i1\"")); + } + + #[test] + fn epub_has_stored_first_mimetype_nav_spine_and_real_images() { + let bytes = epub(&[fixture(), fixture()], "Report & notes", "de-AT").unwrap(); + assert_eq!(&bytes[8..10], &[0, 0]); + assert_eq!(&bytes[28..30], &[0, 0]); + assert_eq!(&bytes[30..38], b"mimetype"); + assert_eq!(&bytes[38..58], b"application/epub+zip"); + let parts = parts(&bytes); + let opf = String::from_utf8(parts["EPUB/package.opf"].clone()).unwrap(); + for expected in ["Report & notes", "de-AT", "idref=\"p1\"", "idref=\"p2\"", "properties=\"nav\""] { + assert!(opf.contains(expected)); + } + let chapter = String::from_utf8(parts["EPUB/page1.xhtml"].clone()).unwrap(); + assert!(chapter.contains("")); + assert!(chapter.contains("日本語")); + assert_eq!(parts["EPUB/images/page1-image1.png"], fixture().images[0].bytes); + assert_eq!(bytes, epub(&[fixture(), fixture()], "Report & notes", "de-AT").unwrap()); + } + + #[test] + fn limits_fail_before_truncating_or_creating_invalid_documents() { + assert!(xlsx(&[], "Empty").is_err()); + assert!(epub(&[fixture()], "Title", "en\" bad").is_err()); + assert!(epub(&[fixture()], "Title", "123").is_err()); + let mut page = fixture(); + page.blocks[0].text = "😀".repeat(16_384); + assert!(xlsx(&[page], "Too long").unwrap_err().contains("32767")); + assert_eq!(column(26), "AA"); + assert_eq!(column(16_383), "XFD"); + } + + #[test] + fn external_validation_fixture_when_requested() { + if let Ok(directory) = std::env::var("PDFCRAFT_EXPORT_VALIDATION_DIR") { + std::fs::write(std::path::Path::new(&directory).join("export.xlsx"), xlsx(&[fixture(), fixture()], "Test report").unwrap()).unwrap(); + std::fs::write(std::path::Path::new(&directory).join("export.epub"), epub(&[fixture(), fixture()], "Test report", "de").unwrap()) + .unwrap(); + } + } +} diff --git a/crates/hot-folder-service/Cargo.toml b/crates/hot-folder-service/Cargo.toml new file mode 100644 index 0000000000..390b0c67a4 --- /dev/null +++ b/crates/hot-folder-service/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "pdfcraft-hot-folder-service" +description = "Explicit operating-system service control for PdfCraft Hot Folder" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } + +[target.'cfg(windows)'.dependencies] +windows-service = "0.8.1" + +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +ctrlc = { version = "3.5", features = ["termination"] } +tempfile = "3" diff --git a/crates/hot-folder-service/examples/service_probe.rs b/crates/hot-folder-service/examples/service_probe.rs new file mode 100644 index 0000000000..2b1b759e4b --- /dev/null +++ b/crates/hot-folder-service/examples/service_probe.rs @@ -0,0 +1,75 @@ +//! Synthetic service lifecycle probe for platform CI; never starts without explicit arguments. +use std::{ + path::Path, + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +fn worker(config: &Path, stopping: &AtomicBool) -> Result<(), String> { + while !stopping.load(Ordering::SeqCst) { + std::fs::write(config.with_extension("heartbeat"), b"running").map_err(|e| e.to_string())?; + std::thread::sleep(Duration::from_millis(50)); + } + std::fs::write(config.with_extension("stopped"), b"stopped gracefully").map_err(|e| e.to_string()) +} + +fn main() -> Result<(), String> { + let mut args = std::env::args().skip(1); + let mut action = "status".to_owned(); + let mut config = None; + let mut account = String::new(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--service" => action = "run".into(), + "--initialize-config" => action = "initialize".into(), + "--config" => config = args.next().map(std::path::PathBuf::from), + "--service-action" => action = args.next().ok_or("missing action")?, + "--account" => account = args.next().ok_or("missing account")?, + _ => return Err("unexpected probe argument".into()), + } + } + let config = config.ok_or("--config is required")?; + let executable = std::env::current_exe().map_err(|e| e.to_string())?; + use pdfcraft_hot_folder_service as service; + match action.as_str() { + "initialize" => { + let parent = config.parent().ok_or("missing configuration directory")?; + let mut directories = std::fs::DirBuilder::new(); + directories.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + directories.mode(0o700); + } + directories.create(parent).map_err(|e| e.to_string())?; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match options.open(&config) { + Ok(mut file) => { + use std::io::Write; + file.write_all(b"{\"tasks\":[]}").map_err(|e| e.to_string())?; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e.to_string()), + } + return Ok(()); + } + "run" => return service::run(&config, worker), + "install" => service::install(&executable, &config, &account)?, + "uninstall" => service::uninstall(&executable, &config)?, + "start" => service::start(&executable, &config)?, + "stop" => service::stop(&executable, &config)?, + "enable" => service::set_enabled(true, &executable, &config)?, + "disable" => service::set_enabled(false, &executable, &config)?, + "status" => {} + _ => return Err("unknown action".into()), + } + let state = service::status(&executable, &config)?; + println!("installed={} enabled={} running={} account={}", state.installed, state.enabled, state.running, state.account); + Ok(()) +} diff --git a/crates/hot-folder-service/src/definitions.rs b/crates/hot-folder-service/src/definitions.rs new file mode 100644 index 0000000000..3a86ec119e --- /dev/null +++ b/crates/hot-folder-service/src/definitions.rs @@ -0,0 +1,134 @@ +//! Service definitions contain arguments as data, never shell commands. + +#[cfg(any(target_os = "linux", target_os = "macos", test))] +pub fn account(value: &str) -> Result<(), String> { + if value.is_empty() + || value.len() > 64 + || value == "root" + || value == "0" + || value.starts_with('-') + || !value.bytes().all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)) + { + return Err("Choose an existing, non-root service account".into()); + } + Ok(()) +} + +#[cfg(any(windows, test))] +pub fn windows_command(executable: &str, config: &str) -> Result { + if [executable, config].iter().any(|s| s.contains('"') || s.chars().any(char::is_control) || s.ends_with('\\')) { + return Err("Service paths cannot contain quotes, control characters or a trailing backslash".into()); + } + Ok(format!("\"{executable}\" --service --config \"{config}\"")) +} + +#[cfg(any(windows, test))] +pub fn windows_command_matches(command: &str, executable: &str, config: &str) -> Result { + let quoted = windows_command(executable, config)?; + // MSI quotes both paths; windows-service quotes only arguments containing spaces. + // Restrict matching to these two known spellings rather than parse arbitrary shell text. + let argument = |value: &str| if value.contains(' ') { format!("\"{value}\"") } else { value.into() }; + Ok(command == quoted || command == format!("{} --service --config {}", argument(executable), argument(config))) +} + +#[cfg(any(target_os = "linux", test))] +fn unit_argument(value: &str) -> String { + format!("\"{}\"", value.replace('\\', "\\\\").replace('"', "\\\"").replace('%', "%%").replace('$', "$$")) +} + +#[cfg(any(target_os = "linux", test))] +pub fn systemd(executable: &str, config: &str, user: &str) -> Result { + account(user)?; + Ok(format!( + "# PdfCraft Hot Folder system service\n[Unit]\nDescription=PdfCraft Hot Folder\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nUser={user}\nExecStart={} --service --config {}\nRestart=on-failure\nRestartSec=10\nTimeoutStopSec=120\nUMask=0077\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=full\n\n[Install]\nWantedBy=multi-user.target\n", + unit_argument(executable), + unit_argument(config) + )) +} + +#[cfg(any(target_os = "macos", test))] +pub fn launchd_enabled(output: &str) -> Result { + let label = format!("\"{}\"", crate::UNIX_LABEL); + for line in output.lines() { + if let Some((key, value)) = line.trim().split_once("=>") + && key.trim() == label + { + return match value.trim() { + "true" | "disabled" => Ok(false), + "false" | "enabled" => Ok(true), + _ => Err("Unrecognized launchd startup state".into()), + }; + } + } + Ok(true) +} + +#[cfg(any(target_os = "macos", test))] +fn xml(value: &str) -> String { + value.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'") +} + +#[cfg(any(target_os = "macos", test))] +pub fn launchd(executable: &str, config: &str, user: &str) -> Result { + account(user)?; + Ok(format!( + "\n\n\nLabel{}\nUserName{}\nProgramArguments{}--service--config{}\nRunAtLoad\nKeepAliveSuccessfulExit\nThrottleInterval10\nExitTimeOut120\nUmask63\nProcessTypeBackground\n\n", + crate::UNIX_LABEL, + xml(user), + xml(executable), + xml(config) + )) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn launchd_reads_both_boolean_and_modern_disabled_output() { + for value in ["true", "disabled"] { + assert!(!launchd_enabled(&format!("disabled services = {{\n\t\"{}\" => {value}\n}}", crate::UNIX_LABEL)).unwrap()); + } + for value in ["false", "enabled"] { + assert!(launchd_enabled(&format!("\"{}\" => {value}", crate::UNIX_LABEL)).unwrap()); + } + assert!(launchd_enabled("\"another.service\" => disabled").unwrap()); + assert!(launchd_enabled(&format!("\"{}\" => unknown", crate::UNIX_LABEL)).is_err()); + } + #[test] + fn generated_units_cannot_inject_arguments_or_run_as_root() { + let text = systemd("/opt/Pdf Craft/worker", "/home/test/A%$B/tasks.json", "test").unwrap(); + assert!(text.contains("User=test\n")); + assert!(text.contains("\"/home/test/A%%$$B/tasks.json\"")); + assert!(text.contains("WantedBy=multi-user.target")); + assert!(systemd("/bin/worker", "/tasks.json", "root").is_err()); + assert!(systemd("/bin/worker", "/tasks.json", "user\nExecStart=bad").is_err()); + let plist = launchd("/Applications/PdfCraft Hot Folder.app/Contents/MacOS/worker", "/Users/A & B/.json", "test").unwrap(); + assert!(plist.contains("A & B/<tasks>.json")); + assert!(plist.contains("UserNametest")); + assert!(!plist.contains("/bin/sh")); + } + #[test] + fn windows_binary_and_config_are_separate_quoted_arguments() { + assert_eq!( + windows_command(r"C:\Program Files\PdfCraft\worker.exe", r"C:\ProgramData\PdfCraft\tasks.json").unwrap(), + r#""C:\Program Files\PdfCraft\worker.exe" --service --config "C:\ProgramData\PdfCraft\tasks.json""# + ); + assert!(windows_command("bad\"exe", "file").is_err()); + assert!( + windows_command_matches( + r#""C:\Program Files\worker.exe" --service --config C:\tasks.json"#, + r"C:\Program Files\worker.exe", + r"C:\tasks.json" + ) + .unwrap() + ); + assert!( + !windows_command_matches( + r"C:\Program Files\worker.exe --service --config C:\tasks.json", + r"C:\Program Files\worker.exe", + r"C:\tasks.json" + ) + .unwrap() + ); + } +} diff --git a/crates/hot-folder-service/src/lib.rs b/crates/hot-folder-service/src/lib.rs new file mode 100644 index 0000000000..01d6888bf6 --- /dev/null +++ b/crates/hot-folder-service/src/lib.rs @@ -0,0 +1,111 @@ +//! Explicit system-boot service management, shared by the companion and automation. +//! No service is installed or started merely by linking this crate. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::path::Path; +use std::sync::atomic::AtomicBool; + +mod definitions; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod unix; +#[cfg(not(any(windows, target_os = "linux", target_os = "macos")))] +mod unsupported; +#[cfg(windows)] +mod windows; + +pub const SERVICE_NAME: &str = "PdfCraftHotFolder"; +pub const UNIX_LABEL: &str = "ai.storyteller.pdfcraft.hot-folder"; +pub type Worker = fn(&Path, &AtomicBool) -> Result<(), String>; + +#[derive(Debug, Clone, serde::Serialize)] +pub struct Status { + pub installed: bool, + pub enabled: bool, + pub running: bool, + pub account: String, +} + +impl Status { + #[cfg(any(windows, target_os = "linux", target_os = "macos"))] + fn absent() -> Self { + Self { installed: false, enabled: false, running: false, account: String::new() } + } +} + +fn path_text(path: &Path) -> Result<&str, String> { + let value = path.to_str().ok_or("Service paths must contain valid Unicode")?; + if !path.is_absolute() || value.chars().any(char::is_control) { + return Err("Service paths must be absolute and contain no control characters".into()); + } + Ok(value) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +use unix as platform; +#[cfg(not(any(windows, target_os = "linux", target_os = "macos")))] +use unsupported as platform; +#[cfg(windows)] +use windows as platform; + +/// Inspect only the registered service matching this executable and task file. +pub fn status(executable: &Path, config: &Path) -> Result { + path_text(executable)?; + path_text(config)?; + platform::status(executable, config) +} + +/// Change boot startup, without changing the service account or task file. +pub fn set_enabled(enabled: bool, executable: &Path, config: &Path) -> Result<(), String> { + if !status(executable, config)?.installed { + return Err("Install the optional Hot Folder system service first".into()); + } + platform::set_enabled(enabled, executable, config) +} + +pub fn start(executable: &Path, config: &Path) -> Result<(), String> { + let state = status(executable, config)?; + if !state.installed { + return Err("The Hot Folder system service is not installed".into()); + } + if !state.running { + platform::start()?; + } + Ok(()) +} + +pub fn stop(executable: &Path, config: &Path) -> Result<(), String> { + if status(executable, config)?.running { + platform::stop()?; + } + Ok(()) +} + +/// Unix requires an explicit non-root account. Windows registration belongs to MSI. +pub fn install(executable: &Path, config: &Path, account: &str) -> Result<(), String> { + path_text(executable)?; + path_text(config)?; + platform::install(executable, config, account) +} + +pub fn uninstall(executable: &Path, config: &Path) -> Result<(), String> { + if status(executable, config)?.installed { + platform::uninstall()?; + } + Ok(()) +} + +/// Blocking service host. The worker must check the cancellation flag between pages. +pub fn run(config: &Path, worker: Worker) -> Result<(), String> { + path_text(config)?; + platform::run(config, worker) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn service_paths_reject_relative_and_control_characters() { + assert!(path_text(Path::new("tasks.json")).is_err()); + assert!(path_text(Path::new("/tmp/tasks\n.json")).is_err()); + } +} diff --git a/crates/hot-folder-service/src/unix.rs b/crates/hot-folder-service/src/unix.rs new file mode 100644 index 0000000000..b38dc8448e --- /dev/null +++ b/crates/hot-folder-service/src/unix.rs @@ -0,0 +1,203 @@ +use crate::{Status, Worker}; +use std::{ + io::Write, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::Path, + process::{Command, Output}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +#[cfg(target_os = "macos")] +const DEFINITION: &str = "/Library/LaunchDaemons/ai.storyteller.pdfcraft.hot-folder.plist"; +#[cfg(not(target_os = "macos"))] +const DEFINITION: &str = "/etc/systemd/system/pdfcraft-hot-folder.service"; + +fn command(program: &str, args: &[&str]) -> Result { + Command::new(program).args(args).output().map_err(|e| format!("Could not run {program}: {e}")) +} +fn checked(program: &str, args: &[&str]) -> Result<(), String> { + let output = command(program, args)?; + if output.status.success() { + Ok(()) + } else { + Err(format!("{program} failed. Administrator permission may be required: {}", String::from_utf8_lossy(&output.stderr).trim())) + } +} + +fn account_uid(account: &str) -> Result { + crate::definitions::account(account)?; + let output = command("/usr/bin/id", &["-u", account])?; + let uid = String::from_utf8_lossy(&output.stdout).trim().parse::().map_err(|_| "The service account does not exist")?; + if !output.status.success() || uid == 0 { + return Err("The service must run under an existing non-root account".into()); + } + Ok(uid) +} + +fn contents(executable: &Path, config: &Path, account: &str) -> Result { + #[cfg(target_os = "macos")] + { + crate::definitions::launchd(crate::path_text(executable)?, crate::path_text(config)?, account) + } + #[cfg(not(target_os = "macos"))] + { + crate::definitions::systemd(crate::path_text(executable)?, crate::path_text(config)?, account) + } +} + +fn read_definition() -> Result, String> { + match std::fs::symlink_metadata(DEFINITION) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.to_string()), + Ok(meta) => { + if !meta.is_file() || meta.uid() != 0 || meta.mode() & 0o022 != 0 || meta.len() > 16384 { + return Err("The service definition must be a root-owned file, not writable by other accounts".into()); + } + std::fs::read_to_string(DEFINITION).map(Some).map_err(|e| e.to_string()) + } + } +} + +fn registered_account(text: &str) -> Result<&str, String> { + #[cfg(target_os = "macos")] + let value = text.split_once("UserName").and_then(|(_, tail)| tail.split_once("")).map(|(value, _)| value); + #[cfg(not(target_os = "macos"))] + let value = text.lines().find_map(|line| line.strip_prefix("User=")); + value.ok_or_else(|| "The installed service definition is not managed by PdfCraft".into()) +} + +pub fn status(executable: &Path, config: &Path) -> Result { + let Some(text) = read_definition()? else { + return Ok(Status::absent()); + }; + let account = registered_account(&text)?; + if text != contents(executable, config, account)? { + return Err("The installed service uses a different executable or task file, or has been edited externally".into()); + } + #[cfg(target_os = "macos")] + let (enabled, running) = { + let disabled = command("/bin/launchctl", &["print-disabled", "system"])?; + if !disabled.status.success() { + return Err("Could not read launchd startup state".into()); + } + let enabled = crate::definitions::launchd_enabled(&String::from_utf8_lossy(&disabled.stdout))?; + let state = command("/bin/launchctl", &["print", &format!("system/{}", crate::UNIX_LABEL)])?; + (enabled, state.status.success() && String::from_utf8_lossy(&state.stdout).contains("state = running")) + }; + #[cfg(not(target_os = "macos"))] + let (enabled, running) = { + let enabled = command("/usr/bin/systemctl", &["is-enabled", "pdfcraft-hot-folder.service"])?; + let running = command("/usr/bin/systemctl", &["is-active", "pdfcraft-hot-folder.service"])?; + (enabled.status.success(), running.status.success()) + }; + Ok(Status { installed: true, enabled, running, account: account.into() }) +} + +pub fn set_enabled(enabled: bool, _: &Path, _: &Path) -> Result<(), String> { + change_startup(enabled) +} +fn change_startup(enabled: bool) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + checked("/bin/launchctl", &[if enabled { "enable" } else { "disable" }, &format!("system/{}", crate::UNIX_LABEL)]) + } + #[cfg(not(target_os = "macos"))] + { + checked("/usr/bin/systemctl", &[if enabled { "enable" } else { "disable" }, "pdfcraft-hot-folder.service"]) + } +} +pub fn start() -> Result<(), String> { + #[cfg(target_os = "macos")] + { + let loaded = command("/bin/launchctl", &["print", &format!("system/{}", crate::UNIX_LABEL)])?; + if loaded.status.success() { + checked("/bin/launchctl", &["kickstart", &format!("system/{}", crate::UNIX_LABEL)]) + } else { + checked("/bin/launchctl", &["bootstrap", "system", DEFINITION]) + } + } + #[cfg(not(target_os = "macos"))] + { + checked("/usr/bin/systemctl", &["start", "pdfcraft-hot-folder.service"]) + } +} +pub fn stop() -> Result<(), String> { + #[cfg(target_os = "macos")] + { + checked("/bin/launchctl", &["bootout", &format!("system/{}", crate::UNIX_LABEL)]) + } + #[cfg(not(target_os = "macos"))] + { + checked("/usr/bin/systemctl", &["stop", "pdfcraft-hot-folder.service"]) + } +} + +pub fn install(executable: &Path, config: &Path, account: &str) -> Result<(), String> { + let uid = account_uid(account)?; + // System startup must never execute a binary replaceable by a less-privileged user. + let executable = std::fs::canonicalize(executable).map_err(|e| e.to_string())?; + for ancestor in executable.ancestors() { + let meta = std::fs::metadata(ancestor).map_err(|e| e.to_string())?; + let trusted_admin_group = cfg!(target_os = "macos") && meta.gid() == 80; + if meta.uid() != 0 || meta.mode() & 0o002 != 0 || (meta.mode() & 0o020 != 0 && !trusted_admin_group) { + return Err(format!( + "Install the executable in a root-owned directory without group/other write access before registering its service: {}", + ancestor.display() + )); + } + } + let config_meta = std::fs::symlink_metadata(config).map_err(|e| format!("Create the task file for the service account first: {e}"))?; + if !config_meta.is_file() || config_meta.uid() != uid || config_meta.mode() & 0o077 != 0 { + return Err("The task file must be owned by the selected service account with owner-only permissions (chmod 600)".into()); + } + let text = contents(&executable, config, account)?; + if let Some(previous) = read_definition()? { + if previous != text { + return Err("A different Hot Folder system service is already installed; remove it before changing its account or paths".into()); + } + } else { + let parent = Path::new(DEFINITION).parent().ok_or("The service directory is unavailable")?; + let mut temporary = + tempfile::NamedTempFile::new_in(parent).map_err(|e| format!("Administrator permission is required to install the service: {e}"))?; + temporary.write_all(text.as_bytes()).map_err(|e| e.to_string())?; + temporary.as_file().set_permissions(std::fs::Permissions::from_mode(0o644)).map_err(|e| e.to_string())?; + temporary.as_file().sync_all().map_err(|e| e.to_string())?; + temporary.persist_noclobber(DEFINITION).map_err(|e| e.to_string())?; + } + #[cfg(not(target_os = "macos"))] + checked("/usr/bin/systemctl", &["daemon-reload"])?; + change_startup(true)?; + start() +} + +pub fn uninstall() -> Result<(), String> { + change_startup(false)?; + // It is safe to remove only our fixed definition, never the user's tasks or inputs. + #[cfg(not(target_os = "macos"))] + stop()?; + #[cfg(target_os = "macos")] + { + if command("/bin/launchctl", &["print", &format!("system/{}", crate::UNIX_LABEL)])?.status.success() { + stop()?; + } + } + std::fs::remove_file(DEFINITION).map_err(|e| e.to_string())?; + #[cfg(not(target_os = "macos"))] + checked("/usr/bin/systemctl", &["daemon-reload"])?; + Ok(()) +} + +pub fn run(config: &Path, worker: Worker) -> Result<(), String> { + let identity = command("/usr/bin/id", &["-u"])?; + let uid = String::from_utf8_lossy(&identity.stdout).trim().parse::().map_err(|_| "Could not determine the service account")?; + if !identity.status.success() || uid == 0 { + return Err("The Hot Folder worker must run under a non-root account; use the installed system service".into()); + } + let stopping = Arc::new(AtomicBool::new(false)); + let signal = stopping.clone(); + ctrlc::set_handler(move || signal.store(true, Ordering::SeqCst)).map_err(|e| e.to_string())?; + worker(config, &stopping) +} diff --git a/crates/hot-folder-service/src/unsupported.rs b/crates/hot-folder-service/src/unsupported.rs new file mode 100644 index 0000000000..e35c6e193a --- /dev/null +++ b/crates/hot-folder-service/src/unsupported.rs @@ -0,0 +1,27 @@ +use crate::{Status, Worker}; +use std::path::Path; + +fn unavailable() -> Result { + Err("System services are supported on Windows, macOS and Linux with systemd".into()) +} +pub fn status(_: &Path, _: &Path) -> Result { + unavailable() +} +pub fn set_enabled(_: bool, _: &Path, _: &Path) -> Result<(), String> { + unavailable() +} +pub fn start() -> Result<(), String> { + unavailable() +} +pub fn stop() -> Result<(), String> { + unavailable() +} +pub fn install(_: &Path, _: &Path, _: &str) -> Result<(), String> { + unavailable() +} +pub fn uninstall() -> Result<(), String> { + unavailable() +} +pub fn run(_: &Path, _: Worker) -> Result<(), String> { + unavailable() +} diff --git a/crates/hot-folder-service/src/windows.rs b/crates/hot-folder-service/src/windows.rs new file mode 100644 index 0000000000..4d11882d0c --- /dev/null +++ b/crates/hot-folder-service/src/windows.rs @@ -0,0 +1,157 @@ +use crate::{SERVICE_NAME, Status, Worker}; +use std::path::{Path, PathBuf}; +use std::sync::{ + OnceLock, + atomic::{AtomicBool, Ordering}, +}; +use std::time::Duration; +use windows_service::{ + service::*, + service_control_handler::{self, ServiceControlHandlerResult}, + service_dispatcher, + service_manager::*, +}; + +fn service(access: ServiceAccess) -> Result { + ServiceManager::local_computer(None::<&str>, ServiceManagerAccess::CONNECT)?.open_service(SERVICE_NAME, access) +} + +pub fn status(executable: &Path, config: &Path) -> Result { + let handle = match service(ServiceAccess::QUERY_STATUS | ServiceAccess::QUERY_CONFIG) { + Ok(handle) => handle, + Err(windows_service::Error::Winapi(error)) if error.raw_os_error() == Some(1060) => return Ok(Status::absent()), + Err(error) => return Err(format!("Could not inspect the Windows service: {error}")), + }; + let registered = handle.query_config().map_err(|e| e.to_string())?; + if !crate::definitions::windows_command_matches( + ®istered.executable_path.to_string_lossy(), + crate::path_text(executable)?, + crate::path_text(config)?, + )? { + return Err("The installed Hot Folder service uses a different executable or configuration; open its configured task file".into()); + } + let state = handle.query_status().map_err(|e| e.to_string())?; + Ok(Status { + installed: true, + enabled: registered.start_type == ServiceStartType::AutoStart, + running: matches!(state.current_state, ServiceState::Running | ServiceState::StartPending), + account: registered.account_name.map(|value| value.to_string_lossy().into_owned()).unwrap_or_else(|| "LocalSystem".into()), + }) +} + +pub fn set_enabled(enabled: bool, executable: &Path, config: &Path) -> Result<(), String> { + let handle = service(ServiceAccess::QUERY_CONFIG | ServiceAccess::CHANGE_CONFIG) + .map_err(|e| format!("Administrator permission is required to change service startup: {e}"))?; + let current = handle.query_config().map_err(|e| e.to_string())?; + handle + .change_config(&ServiceInfo { + name: SERVICE_NAME.into(), + display_name: current.display_name, + service_type: current.service_type, + start_type: if enabled { ServiceStartType::AutoStart } else { ServiceStartType::OnDemand }, + error_control: current.error_control, + executable_path: executable.into(), + launch_arguments: vec!["--service".into(), "--config".into(), config.as_os_str().into()], + dependencies: current.dependencies, + // ChangeServiceConfigW treats a null account/password as "do not change". + account_name: None, + account_password: None, + }) + .map_err(|e| e.to_string()) +} + +pub fn start() -> Result<(), String> { + service(ServiceAccess::START) + .and_then(|s| s.start::<&str>(&[])) + .map_err(|e| format!("Could not start the service (administrator permission may be required): {e}")) +} +pub fn stop() -> Result<(), String> { + service(ServiceAccess::STOP) + .and_then(|s| s.stop()) + .map(|_| ()) + .map_err(|e| format!("Could not stop the service (administrator permission may be required): {e}")) +} +pub fn install(_: &Path, _: &Path, _: &str) -> Result<(), String> { + Err("Install the optional Hot Folder component with the PdfCraft MSI. It registers a LocalService account and a protected task directory.".into()) +} +pub fn uninstall() -> Result<(), String> { + Err("Remove the optional Hot Folder component with the PdfCraft installer, which also removes its Windows service.".into()) +} + +static JOB: OnceLock<(PathBuf, Worker)> = OnceLock::new(); +static OUTCOME: OnceLock> = OnceLock::new(); +windows_service::define_windows_service!(service_entry, service_main); + +pub fn run(config: &Path, worker: Worker) -> Result<(), String> { + JOB.set((config.to_owned(), worker)).map_err(|_| "The Windows service host can only start once per process")?; + service_dispatcher::start(SERVICE_NAME, service_entry).map_err(|e| format!("Start this mode through Windows Services: {e}"))?; + OUTCOME.get().cloned().unwrap_or_else(|| Err("The service did not report its completion".into())) +} + +fn service_main(_: Vec) { + let result = std::panic::catch_unwind(host).unwrap_or_else(|_| Err("The service worker failed safely".into())); + let _ = OUTCOME.set(result); +} + +fn status_value(state: ServiceState, failed: bool) -> ServiceStatus { + ServiceStatus { + service_type: ServiceType::OWN_PROCESS, + current_state: state, + controls_accepted: if state == ServiceState::Running { + ServiceControlAccept::STOP | ServiceControlAccept::SHUTDOWN + } else { + ServiceControlAccept::empty() + }, + exit_code: if failed { ServiceExitCode::ServiceSpecific(1) } else { ServiceExitCode::Win32(0) }, + checkpoint: 0, + wait_hint: Duration::default(), + process_id: None, + } +} + +fn host() -> Result<(), String> { + let stopping = std::sync::Arc::new(AtomicBool::new(false)); + let signal = stopping.clone(); + let handle = service_control_handler::register(SERVICE_NAME, move |event| match event { + ServiceControl::Stop | ServiceControl::Shutdown => { + signal.store(true, Ordering::SeqCst); + ServiceControlHandlerResult::NoError + } + ServiceControl::Interrogate => ServiceControlHandlerResult::NoError, + _ => ServiceControlHandlerResult::NotImplemented, + }) + .map_err(|e| e.to_string())?; + handle.set_service_status(status_value(ServiceState::Running, false)).map_err(|e| e.to_string())?; + let finished = AtomicBool::new(false); + let result = std::thread::scope(|scope| { + // Report cancellation while the current page finishes. Bounded wait hints prevent + // Windows from treating a long OCR page as an unresponsive stop request. + let monitor = std::thread::Builder::new() + .name("service-stop-status".into()) + .spawn_scoped(scope, || { + let mut checkpoint = 0u32; + while !finished.load(Ordering::SeqCst) { + if stopping.load(Ordering::SeqCst) { + checkpoint = checkpoint.saturating_add(1); + let mut state = status_value(ServiceState::StopPending, false); + state.checkpoint = checkpoint; + state.wait_hint = Duration::from_secs(120); + let _ = handle.set_service_status(state); + } + std::thread::sleep(Duration::from_millis(250)); + } + }) + .map_err(|e| format!("Could not start the service status monitor: {e}"))?; + let result = match JOB.get() { + Some((config, worker)) => { + std::panic::catch_unwind(|| worker(config, &stopping)).unwrap_or_else(|_| Err("The service worker failed safely".into())) + } + None => Err("Service configuration is unavailable".into()), + }; + finished.store(true, Ordering::SeqCst); + monitor.join().map_err(|_| "Service status reporting failed safely")?; + result + }); + handle.set_service_status(status_value(ServiceState::Stopped, result.is_err())).map_err(|e| e.to_string())?; + result +} diff --git a/docs/hot-folder-analysis.md b/docs/hot-folder-analysis.md new file mode 100644 index 0000000000..97ca688558 --- /dev/null +++ b/docs/hot-folder-analysis.md @@ -0,0 +1,57 @@ +# PDF analysis reports + +`hot_folder_analysis::analyze(bytes, pages, should_stop)` returns UTF-8 JSON with +`format: "pdfcraft-analysis"` and `version: 1`. It reads existing PDF text and +drawing instructions. It never runs OCR, document scripts, rendering or a save. +An empty page selection means all pages; otherwise zero-based page indexes retain +the caller's order. Duplicates and invalid indexes are rejected. + +The document section records PDF version, byte size, page count, encryption, +standard Document Info text fields, XMP presence and parser repair notices. Each +selected page records media/crop boxes, normalized rotation, UserUnit, displayed +size in points, text blocks and image placements. Images inside Form XObjects and +inline images are included. No embedded image bytes are copied into the report. + +Rectangles are `[x0, y0, x1, y1]` in unrotated PDF default user space, with the y +axis upward. Multiply by the page's `user_unit` for physical points; apply its +crop and rotation separately for displayed coordinates. Image matrices map an +image unit square into that same PDF coordinate system. Text font size is the +existing text-space size, not a measured raster size. + +`content` distinguishes: + +- `existing_text`: at least one nonempty text block was decoded, including an + already existing hidden searchable text layer. +- `image_without_text`: image drawing instructions exist but no text was decoded. + This is a possible scan, not proof that the image contains text. +- `no_extractable_text`: text/vector/shading instructions exist without decoded + text. Missing font mappings are reported rather than silently treated as blank. +- `empty_content`: no analyzed painting/text instructions were found. Annotation + appearances and dynamic forms are not included, so this does not certify that + a rendered page is visually blank. + +Text blocks follow existing content order, which may differ from human reading +order in complex layouts. Image rectangles are placement bounds before clipping, +masks, transparency or optional-layer visibility. The report does not infer +tables, detect semantic regions, parse XMP fields, or visually classify scanned +pages. These limitations are also included in the JSON itself. + +The operation rejects reports over 16 MiB, decoded text over 8 MiB and more than +50,000 reported regions. Input is limited to 512 MiB and 10,000 pages; each selected +page permits 8 MiB expanded content, 2 MiB per stream, 100,000 operators, 256 form +visits, form depth 12 and graphics-state depth 256. Cycles and malformed content +return an error instead of publishing a silently truncated report. Extraction +permissions are honored; the API does not take a password to unlock protected +input. Cancellation is checked between traversal/extraction phases and content +operators; underlying parser/font operations have their existing bounded work. + +Call this synchronous API from a worker. In a Hot Folder task, analysis-only mode +must suppress OCR and scan transformations before analysis. A report accompanying +a processed-PDF export describes those processed bytes, including any text layer +explicitly added by the task's earlier OCR step. + +Validation: `cargo test -p pdfcraft-engine hot_folder_analysis --lib` covers Unicode +metadata, text/image/empty/vector pages, nested and inline image coordinates, +Rotate/UserUnit, deterministic output, selection order, unchanged input bytes, +cancellation, malformed input, cyclic forms and oversized streams. All fixtures +are generated from original test code without distributed document assets. diff --git a/docs/hot-folder-installation.md b/docs/hot-folder-installation.md new file mode 100644 index 0000000000..1daaa838bc --- /dev/null +++ b/docs/hot-folder-installation.md @@ -0,0 +1,105 @@ +# Installing the optional Hot Folder service + +Hot Folder has a separate task manager and a background worker. Installing the optional +component registers a system service that starts at boot, before an interactive login. +Its initial configuration has no tasks. Only tasks that you explicitly configure and enable +are processed. Closing the task-manager window does not stop the service. + +The worker runs under a restricted account, not root or LocalSystem. That account needs read +access to each input folder and write access to output/archive folders. Moving or deleting +originals also needs write access to the input folder. Prefer dedicated scan folders. +User-session network mounts and encrypted home folders may become available only after login; +configure storage that is available to the service account for unattended processing. + +## Windows + +Select **Also install PdfCraft Hot Folder** in the MSI. It is unchecked on a new installation. +An upgrade preserves an installed companion unless you deselect it. For unattended installation, +pass `INSTALLHOTFOLDER=1`; pass `INSTALLHOTFOLDER=0` to decline it on a new installation. +The maintenance screen can add or remove the companion without removing PdfCraft. + +The **PdfCraft Hot Folder** Start menu entry opens the task file used by the service: +`%ProgramData%\PdfCraft\HotFolder\tasks.json`. Editing the system configuration or changing +service startup requires administrator permission. The service runs as +`NT AUTHORITY\LocalService`; grant that account access to the folders selected for its tasks. +Do not put watched files in the protected program or service-configuration directories. + +The MSI registers automatic startup and bounded restart-on-failure recovery. Its task directory +allows administrators and LocalService to write; ordinary users may read status. The separate +`secrets` directory is accessible only to administrators, SYSTEM and LocalService. Credentials +must never be written into exported task JSON. Installer removal stops and unregisters the +service and removes its executable, but retains nonempty task and credential directories. + +The portable ZIP includes the companion executable but does not register a service. Use the +MSI when system boot startup is required. + +## macOS + +Use the `.pkg` installer and select **PdfCraft Hot Folder** in Customize. The selection is off +by default. The standard `.dmg` installs the editor by drag and drop and does not register a +background service. + +The optional package installs `/Applications/PdfCraft Hot Folder.app` and a system LaunchDaemon +at `/Library/LaunchDaemons/ai.storyteller.pdfcraft.hot-folder.plist`. It runs under the local +user signed in during installation, even when that user has not yet opened the task manager. +Tasks are stored in that user's `~/Library/Application Support/PdfCraft/HotFolder/tasks.json`. +Use the app as that user to edit tasks. macOS privacy permissions still govern access to +protected folders; a denied access is reported as a task error. + +The installer stops an old daemon before replacing the app and preserves existing task data. +To remove boot registration while keeping tasks, run this explicit administrative command: + +```sh +sudo '/Applications/PdfCraft Hot Folder.app/Contents/MacOS/PdfCraft Hot Folder' \ + --service-action uninstall \ + --config "$HOME/Library/Application Support/PdfCraft/HotFolder/tasks.json" +``` + +## Linux + +Install the separate `pdfcraft-hot-folder` `.deb` or `.rpm` package alongside the main `pdfcraft` +package. Installing only PdfCraft does not add the service. Hot Folder requires systemd. +The optional package creates the non-login account `pdfcraft-hot-folder`, stores its configuration +in `/var/lib/pdfcraft-hot-folder/tasks.json`, and enables `pdfcraft-hot-folder.service` in the +system service manager. The state directory is owned by that account with mode 0700. + +Use administrative access to manage the service configuration; preserve its service-account +ownership and mode 0600. Provision remote-source credentials under the same service account, +or use the task manager with administrative permission. Administrative provisioning requires +an existing private configuration owned by a non-root service account and creates secrets with +that account's ownership; it does not run the document processor as root. +Grant the service account the necessary access to your input/output folders. Removing the optional +package stops and unregisters the service while preserving task data and its account for reinstall. + +## Inspecting and controlling the service + +The task manager and CLI share the service API. Explicit commands support `status`, `start`, +`stop`, `enable` and `disable`: + +```sh +pdfcraft-hot-folder --service-action status --config /absolute/path/to/tasks.json +``` + +`enable` and `disable` change automatic startup; `start` and `stop` change the current process. +Mutations require the platform's administrative permission. Status refuses to control a service +registered for a different executable or task file. Windows installation/removal belongs to the +MSI. On Linux/macOS, manual registration additionally requires `--account USER`, an existing +non-root account, a protected task file owned by that account, and a root-owned executable. + +`--worker --config FILE` is an explicitly launched foreground worker for portable/development +use; it does not install itself. All modes use the same persistent journal and worker lock. +No network listener or MCP server is started by the installer or background worker. + +## Validation + +Official macOS PKG signing additionally requires a **Developer ID Installer** certificate +(`APPLE_INSTALLER_CERTIFICATE` and `APPLE_INSTALLER_CERTIFICATE_PASSWORD` in release CI), +alongside the application's Developer ID Application certificate. Without signing credentials, +preview apps are ad-hoc signed and their PKG is unsigned and not notarized. + +`.github/workflows/hot-folder-services.yml` exercises the real Windows Service Control Manager, +Linux systemd and macOS launchd with a synthetic worker. It verifies boot registration, account, +start/stop, startup enable/disable and graceful cancellation. Installer tests compile the actual +MSI/PKG/DEB definitions around the synthetic worker and verify optional installation, task +preservation and removal. These run only on disposable CI machines. They do not replace testing +the final OCR executable and signed installer artifacts on each supported architecture. diff --git a/docs/hot-folder-sources.md b/docs/hot-folder-sources.md new file mode 100644 index 0000000000..62798ef147 --- /dev/null +++ b/docs/hot-folder-sources.md @@ -0,0 +1,232 @@ +# Hot Folder source adapters + +Hot Folder can watch a local/network folder directly, or explicitly stage files +from FTP/FTPS, Outlook, or a PDFCraft project into a task's local inbox. No source +is connected, authenticated or downloaded merely by importing a task configuration. +These adapters are native Rust; the browser/WASM build does not include them. + +## Configuration and credentials + +`SourceConfig` is a tagged JSON object. Its default is `{"kind":"local"}`. Remote +configurations contain a credential **identifier**, never passwords, device secrets, +access tokens or refresh tokens. Configuration export is not a credential backup. +Source actions default to `{"kind":"keep"}`. `move` and `delete` require an explicit +task setting and a durable successful output receipt for the exact source SHA-256. + +Services derive `CredentialStore::for_config(config_path, root)` from their selected +configuration. Its location is `/secrets`. It does not fall +back to a GUI user's OS vault. Provision credentials as the account that runs the +service, or as an elevated Windows administrator where the installer authorizes it: + +| Platform | Required credential storage | +| --- | --- | +| Windows service | `%ProgramData%/PdfCraft/HotFolder/secrets`; protected DACL with only SYSTEM, Administrators and LocalService; owner one of those identities | +| Linux service | Service-account-owned `secrets` directory, mode `0700`; files `0600`, one hard link; provision as the configured service user | +| macOS LaunchDaemon | Same owner/mode requirements, under the selected service user's configuration directory; provision as that user | + +The Windows installer must create the secrets directory **before** provisioning, +without inheriting the ordinary Users read access of the parent. Required DACL: +`D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;S-1-5-19)`. +The code checks both owner and grants before reading or writing secrets. Directory +symlinks, junctions and linked files are refused. Unix provisioning never changes +an existing directory's ownership or permissions silently. + +An elevated Unix GUI can provision a **non-root** service account by opening its +existing configuration through `CredentialStore::for_config`. This creates an +explicit administrator context bound to that configuration's UID/GID. The regular +configuration must already be `0600`, its parent `0700`, and both must have the +same service owner/group. Each operation rechecks that binding; a missing, +re-owned, linked or broadly readable configuration is rejected. New secret files, +temporary generations and the lock are assigned to the bound UID/GID **before** +secret contents are written, and remain `0600`. An absent secrets directory can +be created as `0700` for exactly that owner. Existing files are never silently +re-owned. This context is for provisioning, not for running the OCR worker as root. + +Service secrets are permission-protected files, not application-level encrypted +files. OS access control is their security boundary. Per-user OS vaults are also +available through an explicit `CredentialStore::OsVault` execution context for +portable, interactive workers; they are unsuitable for an unrelated boot service. +The credential-store context is not serialized into task configuration. + +Secret changes use generation-tagged chunks and an atomic verified pointer, with +a cross-process file lock in service mode. A failed refresh leaves the previous +complete generation available. Secret buffers are zeroized when dropped where +the API owns them; service files must be included only in protected system backups. +Error messages never include passwords, tokens or raw Graph response bodies. + +## FTP and explicit FTPS + +```json +{ + "kind": "ftp", + "host": "scanner.example.org", + "port": 21, + "security": "explicit_tls", + "username": "scans", + "credential_id": "office-scanner", + "folder": "/incoming", + "recursive": false, + "action": {"kind": "move", "destination": "/processed"} +} +``` + +Save the password separately with `save_ftp_password(store, id, password)`. +`explicit_tls` is the default. It requires `AUTH TLS`, validates the certificate +and hostname against OS trust roots, and protects data connections with `PROT P`. +There is no certificate-ignore option and no fallback to clear text. Plain FTP is +available only through explicit `security: "plain"`; it exposes credentials and +contents to the network. SFTP and implicit FTPS on port 990 are not implemented. + +The server must provide UTF-8 RFC 3659 `MLSD` listings with modification times. +Binary passive transfers use EPSV, with a PASV fallback, always to the control +connection's peer address. Server-provided alternate data addresses are ignored. +Active FTP, symlink entries and directory traversal are refused. Recursion is +limited to 16 levels and enumeration to 10,000 entries. Archive directories must +already exist outside the watched directory. + +Each transfer checks metadata before and after downloading. Before moving or +deleting an original the adapter checks its version and re-downloads its hash. +Archive names have a stable source-ID prefix; an occupied, different target is +never intentionally replaced. FTP has no portable atomic compare-and-delete or +no-clobber rename primitive: a server-side concurrent writer can still race the +final command. Use an inbox whose completed uploads are not modified by other +writers; keeping originals avoids this protocol limitation. + +## Outlook via Microsoft Graph + +Register your own Microsoft Entra public-client application and enable the device +authorization flow. Choose supported account types suitable for your account; +there is no borrowed or bundled application/client ID. Organization consent or +conditional-access policy may prevent device login. + +```json +{ + "kind": "outlook", + "client_id": "YOUR-OWN-APPLICATION-GUID", + "tenant": "common", + "credential_id": "office-mail", + "folder_id": "inbox", + "action": {"kind": "keep"} +} +``` + +An explicit sign-in returns Microsoft's verification URL, user code and expiry. +The device secret is held only in the selected protected credential store. +Polling respects the server interval, `slow_down`, denial and expiry. Refresh +tokens support unattended work after the user authorizes the application. Refresh +checks the account ID and refuses a change of mailbox. Revocation, expired consent +or organizational restrictions require a new explicit sign-in. + +Scopes are `offline_access User.Read Mail.Read` when keeping originals and +`offline_access User.Read Mail.ReadWrite` when moving/deleting. Changing between +these permission sets requires another sign-in. Mail folders can be listed and +selected by ID, including child folders. Only Microsoft's global Graph cloud is +supported; national-cloud Graph endpoints and on-premises Exchange are not. + +The adapter downloads non-inline PDF, PNG, JPEG and TIFF **file attachments**. +Cloud/reference attachments, attached messages and inline signatures are skipped. +Immutable message IDs survive moves within the same mailbox. Every source version +records mailbox identity, immutable message ID, attachment ID, modification time +and size. Pagination and redirects cannot send bearer credentials to another host. + +Original actions affect the **whole email**, never individual attachments. All +supported attachments in the email must have confirmed output receipts. The +adapter fetches the complete current attachment list and verifies every hash +before moving the email. Local PDF/image filters that exclude an attachment can +therefore leave the message in its original folder; they never authorize losing +an unprocessed attachment. An email with more than 128 supported attachments or +more than 512 MiB of supported attachments is rejected intact. + +`delete` moves the email to **Deleted Items**, using Graph's move API; it never +permanently deletes the message. Move/delete retries first reconcile the immutable +message's current folder. An uncertain POST is not blindly retried. If a message +or attachment changes after processing, the adapter retains it and reports the +conflict. Concurrent external edits between the final check and move are not +transactionally locked by Graph; use a dedicated intake folder. + +## Open project manifests + +A PDFCraft project is a UTF-8 JSON manifest, not a proprietary ABBYY OCR project. +The manifest keeps the requested input order, including after a restart: + +```json +{ + "format": "pdfcraft-hot-folder-project", + "version": 1, + "files": ["chapter-2/scan.pdf", "chapter-1/scan.png"] +} +``` + +Select it with `{"kind":"project","manifest":"/absolute/project.json"}`. +Paths in `files` are unique relative paths beneath the manifest directory, with +no `..`, absolute paths or symlink/junction components. All paths honor `--root`. +Missing, never-staged sources are errors, so a grouped export cannot silently +omit them. Manifest entries remain after confirmed move/delete actions. Projects +support the same PDF/PNG/JPEG/TIFF inputs; Office files and ABBYY project formats +are not accepted. + +## Worker integration, restart and limits + +Call `validate` and `local_paths` during cross-task validation. The latter returns +project originals for overlap checks against other task inputs, archives and +outputs. Call blocking `stage` and `acknowledge` only on a worker thread. + +`stage` writes only into the selected inbox, using temporary files, SHA-256 checks +and atomic no-clobber publication. It returns both new and previously staged inputs. +`relative` is the logical source path used for grouping; `staged_relative` is the +flat filename used for reading. Configuration changes cannot reuse another +configuration's journal: choose a fresh staging inbox. + +The core must durably save **all** configured outputs and journal their source hash +before calling `acknowledge`. Acknowledgement records readiness before any remote +action. Failed remote actions remain in `pending_acknowledgements`; the core must +recheck its own durable output receipt before retrying them. Ready sources are not +downloaded again. A missing local stage file after readiness is allowed. The +adapter never removes staging files itself. + +Limits: 512 MiB per file and per stage batch, 128 files per batch, 10,000 listed +entries/journal versions, 8 MiB metadata responses, 200 Graph pagination requests, +five-minute operation deadline. `has_more` signals a truncated batch; Folder/All +grouping must wait/report the limit rather than publish a partial group. Per-file +tasks can advance through successive batches. A journal reaching its version limit +requires a fresh task/inbox; journal data is never silently discarded. + +Cancellation is checked between source operations and transfer blocks. FTP socket +operations time out after ten seconds; Graph HTTP operations after 45 seconds. +TLS internal reads also enforce the overall deadline, including slowly sending +peers. Cancellation does not undo an already confirmed server action; the durable +journal reconciles it on restart. Partially downloaded files are not published. +On Unix the source journal and project archive directory entries are synced after +atomic publication, before proceeding to an original-file action. A directory +sync failure stops the operation; its journal can be reconciled on retry. + +## Validation evidence and boundaries + +`cargo test -p pdfcraft-engine hot_folder_sources --lib` exercises synthetic local +project files, durable receipts, cancellation, changed originals, no-clobber +archives, queue truncation, missing entries and manifest order. Loopback socket +tests execute real passive FTP listing/download/move and TLS trust/hostname checks. +Graph transport fixtures exercise device polling, refresh/account binding, +pagination confinement, whole-email completeness, content verification and retry +reconciliation. Credential tests cover generation integrity, permission refusal +and the Windows protected-ACL policy. Test certificates are generated at runtime; +no third-party scan, certificate, account or mailbox is distributed as a fixture. + +These tests do not constitute a live Microsoft tenant, production FTP deployment +or cross-platform installer acceptance test. No contributor account is connected +by the test suite. Service startup, account consent and target-platform credential +provisioning should be checked in the installer acceptance environment. + +## Protocol references + +- [Linux fsync and directory-entry persistence](https://man7.org/linux/man-pages/man2/fsync.2.html) +- [RFC 2428: EPSV](https://www.rfc-editor.org/rfc/rfc2428.html) +- [RFC 3659: MLSD facts](https://www.rfc-editor.org/info/rfc3659/) +- [RFC 4217: FTP over TLS](https://www.rfc-editor.org/info/rfc4217/) +- [Microsoft device authorization grant](https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-device-code) +- [Outlook immutable identifiers](https://learn.microsoft.com/en-us/graph/outlook-immutable-id) +- [List message attachments](https://learn.microsoft.com/en-us/graph/api/message-list-attachments?view=graph-rest-1.0) +- [Read attachment content](https://learn.microsoft.com/en-us/graph/api/attachment-get?view=graph-rest-1.0) +- [List mail folders](https://learn.microsoft.com/en-us/graph/api/user-list-mailfolders?view=graph-rest-1.0) +- [List messages in a folder](https://learn.microsoft.com/en-us/graph/api/mailfolder-list-messages?view=graph-rest-1.0) +- [Move a message](https://learn.microsoft.com/en-us/graph/api/message-move?view=graph-rest-1.0) diff --git a/docs/hot-folder.md b/docs/hot-folder.md new file mode 100644 index 0000000000..b3ad2c548c --- /dev/null +++ b/docs/hot-folder.md @@ -0,0 +1,154 @@ +# PdfCraft Hot Folder + +**Status: implemented native workflow, in final integration review. Proprietary ABBYY task/project formats are not compatible.** + +The optional standalone application processes folders without keeping the PdfCraft editor open. +It shares OCR, orientation correction, deskew and paper-edge detection with the engine. Its +interface follows the primary system language when English, German, French, Dutch, Spanish, +Norwegian Bokmål or Swedish is available, otherwise English. The document language defaults +independently from the system language, using the shared OCR language list. + +## Current processing contract + +- Create named tasks with separate existing input/output folders. Local folders and network + folders mounted as filesystem paths use the same scanner; credentials are managed by the OS. +- FTP with explicit TLS (or explicitly selected plain FTP), Outlook attachment folders and open + PDFCraft project manifests use a separate local staging folder. Sources are downloaded atomically; + output receipts precede source acknowledgement. Interrupted acknowledgement retries do not rerun + OCR. Project manifests retain their specified file order. No implicit FTPS or SFTP is advertised. +- Choose PDFs, PNG/JPEG/TIFF/BMP images, optional subfolders, start time, once or recurring checks, + and how long a file must remain unchanged before processing. No recursive symlink traversal. + Daily, weekly and monthly schedules use local wall-clock time. A missing month day uses that + month's last day; a DST gap uses the next valid local minute, and repeated times run once. +- Configure resolution, language, existing-text skipping, recognition, rotation, straightening, + paper-edge cropping and safety padding. Optional blank-page removal is deliberately conservative: + only completely white rendered pages without extracted text are removed; documents with forms, + links or annotations are kept. An entirely blank input requires a separate blank-page archive. +- Configure up to sixteen saving steps: processed or image-only PDF, DOCX, RTF, HTML, text, + PNG, JPEG, TIFF, XLSX, EPUB or a JSON analysis report. Group per file, per containing folder or across a complete settled run. + File/date/time/task/folder/counter templates control names. Each conversion is staged before + publishing. Existing names receive numbered suffixes; byte-identical retries reuse results. + Original move/delete follows success of every saving step and its durable receipt. A failed + grouped batch requires an explicit retry, so missing inputs cannot silently disappear from it. + Receipts retain the hashes of the actual staged export bytes. Missing or replaced outputs stop + source cleanup, including recovery after a crash; older receipts without proofs cannot authorize + deletion. Unix publication synchronizes containing directories before originals are changed. +- Select one-based page ranges and save recognition-area templates in task JSON. Regions use + displayed top-left percentages, include or exclude OCR words, and leave visible content intact. + Optional grayscale, inversion, brightness, contrast, black/white threshold, median noise reduction + and splitting spreads rasterize selected pages at the requested resolution (bounded by renderer + limits). Interactive forms/links/annotations are refused for rasterizing image changes. +- Analyze-only mode writes versioned JSON reports with existing text, geometry, image rectangles and metadata. It bypasses OCR, scan corrections and blank-page removal, and supports the selected page range. An analysis saving step after normal processing describes the resulting PDF; the analyzer itself never runs OCR. +- Optional blank-page archives preserve removed pages as separate PDFs; originals are not removed + until these files also save successfully. An entirely blank input requires an archive destination. +- Durable receipts suppress duplicate processing across worker restarts. Failed files are logged + once, retried when their content changes or through Retry/Start. Stop is checked between pages + and does not save partial OCR output. Editing a running task through automation or its configuration + file also cancels the previous settings snapshot, including a change from Delete to Keep. +- Import/export task JSON, duplicate tasks, inspect the last 1,000 log events, filter failures and + open output folders. Importing never enables tasks automatically. + +The scanner caps input batches at 512 MiB/128 files, enumeration and journal entries at 10,000, nesting at 32, +configuration/state JSON at 16 MiB and tasks at 32. These bounds produce actionable errors. +Files must be closed by their producer before processing; for slow/scattered writes increase the +settling time or have the producer rename the completed file into the input folder. + +## Agent and service interfaces + +`pdfcraft-hot-folder --config tasks.json` opens the separate configuration application. +`pdfcraft-hot-folder --worker --config tasks.json` runs the worker without a window. +`pdfcraft-hot-folder --once --config tasks.json` executes one scheduler pass and prints JSON. +`pdfcraft-hot-folder --service --config tasks.json` hosts the installed system service. +`--service-action status|start|stop|enable|disable|install|uninstall` explicitly administers it. +Unix installation additionally requires `--account USER`, a non-root service account. +Unix processing refuses root; elevated execution is reserved for setup and administration. +`--initialize-config --config PATH` atomically creates a stopped configuration without replacing +an existing file. Unix new directories/files are owner-only. Run it as the service account. +The GUI uses an installed matching service rather than launching a competing worker. Windows +installation/removal uses the optional MSI feature; runtime APIs never install it implicitly. +The worker holds an OS lock so only one process owns a configuration, and each engine pass takes +a second lock shared with agent calls. It does not open a network listener or an MCP server. + +`hot_folder_configure`, `hot_folder_status`, `hot_folder_poll` and `hot_folder_retry` expose the +same engine through `pdfcraft-cli run` and opt-in MCP. All nested paths honour `--root`, including +inputs, every saving/blank archive folder, configuration and sidecars. Poll processes one stable +file or complete bounded batch per task per due interval; it never silently starts a background worker. +`hot_folder_run_now` overrides the next calendar occurrence. `hot_folder_service` exposes explicit +service administration; mutations are refused in a filesystem-confined `--root` session. +`hot_folder_project` writes an ordered open manifest without replacing existing files. +`hot_folder_source` provides credential status/removal, FTP password storage from a named environment +variable, Outlook device sign-in and folder listing. Secrets never appear in tool results or task JSON. + +## Remote credentials and mail handling + +The service uses a separate protected `secrets` directory next to its configuration. Windows +installation restricts this to SYSTEM, Administrators and LocalService. Unix requires the service +account as owner and permissions 0700 for the directory and 0600 for files. Unsafe permissions, +symlinks and missing provisioning are rejected instead of falling back to an unprotected file. +Set up credentials under the service identity, or use an administrator task-manager session +bound to the existing protected service configuration. Unix administrative writes retain the +non-root service account's UID/GID and private file modes. Importing +tasks transfers only credential IDs, never passwords or tokens. The engine also has an explicit OS-vault +API for callers running in an interactive user session; the boot service does not depend on a login keychain. + +Outlook uses Microsoft Graph device authorization with your own public-client application registration. +Keep requests Mail.Read; moving/deleting requests Mail.ReadWrite; offline_access and User.Read retain +the session and verify mailbox identity. Open the displayed Microsoft sign-in URL and enter its code. +The application lists folders after authorization. It processes PDF, PNG, JPEG and TIFF attachments; +cloud-link attachments and attached messages are not treated as documents. Move and Delete apply to +the whole message only after all supported attachments succeed, so enable both PDF and images. +Delete moves messages to Deleted Items, never permanently deletes them. Sources with unsupported or +failed attachments are reported and retained. Server revisions, attachment hashes and mailbox identity +are rechecked before any remote move/delete. + +Example task (folders must already exist): + +```json +{ + "tasks": [{ + "id": "school-scans", + "name": "School scans", + "input": "/scans/in", + "output": "/scans/out", + "enabled": false, + "recursive": true, + "interval_seconds": 10, + "settle_seconds": 3, + "originals": "keep", + "settings": { "language": "de", "auto_crop": true } + }] +} +``` + +`tasks.state.json` stores receipts, queue observations and logs. `tasks.lock` and +`tasks.worker.lock` are OS locks, released automatically after a crash. Corrupt JSON is reported +without resetting or overwriting it. Do not delete the journal unless you intentionally want +to process inputs again. Keep it together with its configuration when archiving a completed job. + +## Functional reference and remaining scope + +The clean-room reference is ABBYY's [Hot Folder documentation](https://help.abbyy.com/en-us/finereader/16/user_guide/hotfolder/). +No ABBYY assets or code are included. A processing scheduler and task settings are the relevant +functional behaviors; this document does not claim compatibility with ABBYY project/task files. + +The functional inventory and its implementation boundaries are: + +| Requirement | Current state | +|---|---| +| Windows/macOS/Linux service at system boot | Service host and explicit administration integrated; separate optional installers have platform lifecycle tests | +| Once / recurring interval and local start time, daily/weekly/monthly | Implemented; each due run drains its stable input queue before scheduling the next calendar occurrence | +| Local and OS-mounted network folders | Implemented | +| FTP and Outlook sources | Authenticated adapters, atomic staging, guarded acknowledgements, device sign-in and folder selection implemented | +| Project source | Ordered open PDFCraft manifests implemented; proprietary ABBYY project formats are not supported | +| OCR / scan corrections / exact searchable PDF | Implemented through existing engine | +| Saved region templates / preprocessing | Page selection, include/exclude regions, grayscale/inversion/threshold/brightness/contrast/median filter and spread splitting implemented; bounded read-only JSON analysis reports available | +| Blank-page removal and separate blank-page archive | Implemented with conservative white-only detection and durable archive receipts | +| Multiple save steps/formats | Eleven document/image formats plus versioned JSON analysis, multiple saving steps, XLSX table cells/images and EPUB book structure implemented | +| Group output by file, folder or entire task | Implemented with a 128-file/512-MiB batch bound and staged publication | +| Import/export, duplicate, stop, retry, logs and failure filtering | Implemented for native task JSON | +| Seven requested UI languages | Complete label catalogs and system-language fallback implemented | + +Do not run a privileged root/LocalSystem worker against configuration editable by unprivileged +users. System-service installation must establish an appropriately restricted account and +configuration ownership. Installing the optional component must not process any folder until +a user explicitly enables a task. diff --git a/docs/roadmap.md b/docs/roadmap.md index 8c2ef7eef0..977f725492 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -77,7 +77,7 @@ total matches the parity estimate. | M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
    Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; Create PDF from multiple files (PDFs, images and text, edited as pages in the grid). Done since: German/French/Spanish/Italian/Dutch/Danish/Norwegian/Swedish/Finnish/Portuguese/horizontal-Japanese OCR and system-language UI defaults, Unicode text layers, automatic quarter-turn correction, bounded deskew, and automatic paper-edge crop with safety padding through UI and automation. Missing: additional OCR languages, deskew with annotations, editable-text OCR output, Office export/import, web printing | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools).
    Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export. Missing: autotag, Tags/Order/Content panels, Reading Order tool, alt-text workflow, compare, geospatial/3D measurement, search index, XFA | -| M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers | +| M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. In final integration review: separate Hot Folder application, durable local/FTP/Outlook/project processing, calendar scheduling, grouped export/analysis, restricted boot services and agent tools; scope in [hot-folder.md](hot-folder.md). Missing: Action Wizard, AI providers | | M14 | 1.0 polish: performance, localization, installers | 120–250 | 15% | 80–150 | Done: PhotoCraft's translation system (`i18n/`: TSV catalogs, `tl!`, command-id and plural entries, system-language detection, strict catalog tests); every dialog, panel and notice goes through `tl!`. Catalogs of ≈ 2,000–2,140 entries for Japanese, Simplified and Traditional Chinese, Russian, Bulgarian, German, Spanish, French, Telugu, Hungarian, Ukrainian, Italian, Brazilian Portuguese and Arabic (logical-order catalog put into display order; layout not yet mirrored); Czech covers the menus. Signed and notarized macOS builds, signed Windows installers (x64, x86, ARM64), AppImage, deb, rpm, Flatpak and FreeBSD packages. #307 (bounded large-PDF memory): shared resource indexes and parser metadata, lazy inspection, bounded caches; a 9,156-page file inspects in ≈ 1.8 s at ≈ 500 MiB instead of exceeding 10 GiB. Shortcut display uses localized modifier/key names across menus, toolbar, palette, Help and tooltips (#496). Missing: a right-to-left layout, a Chinese UI face in releases, Hindi/Indonesian/Korean/Vietnamese, file-backed input and global memory budgets, performance budgets, keyboard-only operation | | | **Total** | **2,085–3,840** | **≈ 49% ready** | **≈ 770–1,400** | Area view of the same work: [target-app-parity.md](target-app-parity.md#by-feature-area) | diff --git a/packaging/linux/hot-folder-nfpm.yaml b/packaging/linux/hot-folder-nfpm.yaml new file mode 100644 index 0000000000..03859030ea --- /dev/null +++ b/packaging/linux/hot-folder-nfpm.yaml @@ -0,0 +1,39 @@ +name: pdfcraft-hot-folder +arch: ${NFPM_ARCH} +platform: linux +version: ${VERSION} +maintainer: ${PDFCRAFT_MAINTAINER} +description: Optional PdfCraft folder automation with a system-boot background service. +homepage: https://github.com/storytold/pdfcraft +license: MIT OR Apache-2.0 +depends: + - pdfcraft + - systemd + - util-linux +overrides: + deb: + depends: + - pdfcraft + - systemd + - util-linux + - passwd + - libdbus-1-3 + rpm: + depends: + - pdfcraft + - systemd + - util-linux + - shadow-utils + - dbus-libs +contents: + - src: ${HOT_STAGE}/usr/bin/pdfcraft-hot-folder + dst: /usr/bin/pdfcraft-hot-folder + file_info: + mode: 0755 + - src: packaging/linux/hot-folder.desktop + dst: /usr/share/applications/ai.storyteller.pdfcraft.hot-folder.desktop + - src: docs/hot-folder.md + dst: /usr/share/doc/pdfcraft-hot-folder/README.md +scripts: + postinstall: packaging/linux/hot-folder-postinst.sh + preremove: packaging/linux/hot-folder-prerm.sh diff --git a/packaging/linux/hot-folder-postinst.sh b/packaging/linux/hot-folder-postinst.sh new file mode 100755 index 0000000000..118a6b8d74 --- /dev/null +++ b/packaging/linux/hot-folder-postinst.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Installing this separate optional package opts in to a restricted system service. +set -eu +account=pdfcraft-hot-folder +if ! id "$account" >/dev/null 2>&1; then + useradd --system --user-group --home-dir /var/lib/pdfcraft-hot-folder --shell /usr/sbin/nologin "$account" +fi +[ "$(id -u "$account")" -ne 0 ] || { echo 'Hot Folder must not run as root.' >&2; exit 1; } +directory=/var/lib/pdfcraft-hot-folder +[ ! -L "$directory" ] || { echo 'The service state directory must not be a symbolic link.' >&2; exit 1; } +install -d -m 700 -o "$account" -g "$account" "$directory" +runuser -u "$account" -- /usr/bin/pdfcraft-hot-folder --initialize-config --config "$directory/tasks.json" +/usr/bin/pdfcraft-hot-folder --service-action install --account "$account" --config "$directory/tasks.json" diff --git a/packaging/linux/hot-folder-prerm.sh b/packaging/linux/hot-folder-prerm.sh new file mode 100755 index 0000000000..29c91d1754 --- /dev/null +++ b/packaging/linux/hot-folder-prerm.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +# Debian passes "upgrade"; RPM passes 1 for an upgrade. Leave registration for the next version. +case "${1:-remove}" in upgrade|1) systemctl stop pdfcraft-hot-folder.service; exit 0 ;; esac +if [ -f /etc/systemd/system/pdfcraft-hot-folder.service ]; then + /usr/bin/pdfcraft-hot-folder --service-action uninstall --config /var/lib/pdfcraft-hot-folder/tasks.json +fi +# Keep task definitions, receipts and the service account, so a reinstall cannot reprocess files. diff --git a/packaging/linux/hot-folder.desktop b/packaging/linux/hot-folder.desktop new file mode 100644 index 0000000000..8dd5a60be5 --- /dev/null +++ b/packaging/linux/hot-folder.desktop @@ -0,0 +1,8 @@ +[Desktop Entry] +Type=Application +Name=PdfCraft Hot Folder +Comment=Configure automatic OCR and scan processing +Exec=pdfcraft-hot-folder --config /var/lib/pdfcraft-hot-folder/tasks.json +Icon=ai.storyteller.pdfcraft +Terminal=false +Categories=Office;Utility; diff --git a/packaging/linux/nfpm.yaml b/packaging/linux/nfpm.yaml index b20bcb6725..74fca0bc2f 100644 --- a/packaging/linux/nfpm.yaml +++ b/packaging/linux/nfpm.yaml @@ -31,6 +31,7 @@ overrides: depends: - libc6 (>= 2.35) - libgcc-s1 + - libdbus-1-3 - libxkbcommon0 - libvulkan1 | libegl1 recommends: @@ -45,6 +46,7 @@ overrides: - xdg-desktop-portal rpm: depends: + - dbus-libs - libxkbcommon - (vulkan-loader or libglvnd-egl) recommends: diff --git a/packaging/linux/package.sh b/packaging/linux/package.sh index 19e2636889..814096d669 100755 --- a/packaging/linux/package.sh +++ b/packaging/linux/package.sh @@ -49,9 +49,9 @@ echo "==> PdfCraft $VERSION for Linux $ARCH ($FORMATS)" if [ "$SKIP_BUILD" = 0 ]; then if [ -n "${CROSS_TARGET:-}" ]; then - (cd "$ROOT" && cargo build --release --locked -p pdfcraft -p pdfcraft-cli --target "$CROSS_TARGET") + (cd "$ROOT" && cargo build --release --locked -p pdfcraft -p pdfcraft-cli -p pdfcraft-hot-folder --target "$CROSS_TARGET") else - (cd "$ROOT" && cargo build --release --locked -p pdfcraft -p pdfcraft-cli) + (cd "$ROOT" && cargo build --release --locked -p pdfcraft -p pdfcraft-cli -p pdfcraft-hot-folder) fi fi BIN="$CARGO_TARGET_DIR/${CROSS_TARGET:+$CROSS_TARGET/}release" @@ -75,6 +75,12 @@ copy_docs "$STAGE/usr/share/doc/pdfcraft" # OCR models: the app finds them at /../share/pdfcraft/models, so in every format below. stage_models "$STAGE/usr/share/pdfcraft/models" +# A separate package keeps the system service genuinely optional. Models are shared with +# the required main PdfCraft package; the historical OCR add-on is never bundled here. +HOT_STAGE="$WORK/hot-folder" +install -Dm755 "$BIN/pdfcraft-hot-folder" "$HOT_STAGE/usr/bin/pdfcraft-hot-folder" +"${CROSS_COMPILE:-}strip" "$HOT_STAGE/usr/bin/pdfcraft-hot-folder" 2>/dev/null || true + if command -v desktop-file-validate >/dev/null; then desktop-file-validate "$STAGE/usr/share/applications/$APP_ID.desktop" fi @@ -112,8 +118,12 @@ if has deb || has rpm; then export NFPM_ARCH="$DEB_ARCH" # nfpm expands env vars in fields like `version` and `arch`, but not in `contents[].src`. sed "s|\${STAGE}|$STAGE|g" "$HERE/nfpm.yaml" >"$WORK/nfpm.yaml" + sed "s|\${HOT_STAGE}|$HOT_STAGE|g" "$HERE/hot-folder-nfpm.yaml" >"$WORK/hot-folder-nfpm.yaml" for fmt in deb rpm; do - if has "$fmt"; then (cd "$ROOT" && nfpm package -f "$WORK/nfpm.yaml" -p "$fmt" -t "$DIST/$BASENAME.$fmt"); fi + if has "$fmt"; then + (cd "$ROOT" && nfpm package -f "$WORK/nfpm.yaml" -p "$fmt" -t "$DIST/$BASENAME.$fmt") + (cd "$ROOT" && nfpm package -f "$WORK/hot-folder-nfpm.yaml" -p "$fmt" -t "$DIST/pdfcraft-hot-folder-$VERSION-linux-$ARCH.$fmt") + fi done fi diff --git a/packaging/macos/Distribution.xml.in b/packaging/macos/Distribution.xml.in new file mode 100644 index 0000000000..edc7f0cbb2 --- /dev/null +++ b/packaging/macos/Distribution.xml.in @@ -0,0 +1,16 @@ + + + PdfCraft + + + + + + + + + + PdfCraft-component.pkg + HotFolder-component.pkg + diff --git a/packaging/macos/HotFolder-Info.plist.in b/packaging/macos/HotFolder-Info.plist.in new file mode 100644 index 0000000000..d322794803 --- /dev/null +++ b/packaging/macos/HotFolder-Info.plist.in @@ -0,0 +1,21 @@ + + + CFBundleDevelopmentRegionen + CFBundleDisplayNamePdfCraft Hot Folder + CFBundleNamePdfCraft Hot Folder + CFBundleExecutablePdfCraft Hot Folder + CFBundleIdentifierai.storyteller.pdfcraft.hot-folder + CFBundleInfoDictionaryVersion6.0 + CFBundlePackageTypeAPPL + CFBundleShortVersionString@SHORT_VERSION@ + CFBundleVersion@SHORT_VERSION@ + PdfCraftVersion@VERSION@ + PdfCraftBuildCommit@BUILD_SHA@ + CFBundleIconFilePdfCraft + LSMinimumSystemVersion11.0 + NSHighResolutionCapable + NSSupportsAutomaticGraphicsSwitching + NSRequiresAquaSystemAppearance + CFBundleLocalizationsendefrnlesnbsv + NSHumanReadableCopyrightCopyright the PdfCraft authors. MIT OR Apache-2.0. + diff --git a/packaging/macos/hot-folder-scripts/postinstall b/packaging/macos/hot-folder-scripts/postinstall new file mode 100755 index 0000000000..eefdb0cb1b --- /dev/null +++ b/packaging/macos/hot-folder-scripts/postinstall @@ -0,0 +1,16 @@ +#!/bin/sh +# A system LaunchDaemon runs before login, with the chosen user's privileges, never root. +set -eu +account="${PDFCRAFT_HOT_FOLDER_USER:-$(/usr/bin/stat -f '%Su' /dev/console)}" +case "$account" in ''|root|loginwindow|_*|*[!A-Za-z0-9_-]*) + echo 'Hot Folder needs a signed-in non-root user, or PDFCRAFT_HOT_FOLDER_USER for unattended installation.' >&2; exit 1 ;; +esac +uid="$(/usr/bin/id -u "$account")" +[ "$uid" -ne 0 ] || exit 1 +user_home="$(/usr/bin/dscl . -read "/Users/$account" NFSHomeDirectory | /usr/bin/sed 's/^NFSHomeDirectory: //')" +case "$user_home" in /Users/*) ;; *) echo 'The service account needs an absolute local /Users home.' >&2; exit 1 ;; esac +config="$user_home/Library/Application Support/PdfCraft/HotFolder/tasks.json" +executable='/Applications/PdfCraft Hot Folder.app/Contents/MacOS/PdfCraft Hot Folder' +# Never perform privileged writes/chown through a user-controlled home-directory path. +/usr/bin/sudo -H -u "$account" "$executable" --initialize-config --config "$config" +"$executable" --service-action install --account "$account" --config "$config" diff --git a/packaging/macos/hot-folder-scripts/preinstall b/packaging/macos/hot-folder-scripts/preinstall new file mode 100755 index 0000000000..d76cc2971b --- /dev/null +++ b/packaging/macos/hot-folder-scripts/preinstall @@ -0,0 +1,6 @@ +#!/bin/sh +# Stop an existing instance before the optional app is upgraded; keep all task data. +set -eu +if /bin/launchctl print system/ai.storyteller.pdfcraft.hot-folder >/dev/null 2>&1; then + /bin/launchctl bootout system/ai.storyteller.pdfcraft.hot-folder +fi diff --git a/packaging/macos/import-cert.sh b/packaging/macos/import-cert.sh index ae5ee44c6d..b6c560241e 100755 --- a/packaging/macos/import-cert.sh +++ b/packaging/macos/import-cert.sh @@ -3,6 +3,8 @@ # # Inputs (env): APPLE_CERTIFICATE (base64 .p12), APPLE_CERTIFICATE_PASSWORD, KEYCHAIN_PASSWORD # (optional; random if unset), RUNNER_TEMP (or TMPDIR). +# Optional PKG signing: APPLE_INSTALLER_CERTIFICATE (base64 Developer ID Installer .p12) +# and APPLE_INSTALLER_CERTIFICATE_PASSWORD. Output MACOS_INSTALLER_SIGN_IDENTITY. # Outputs: MACOS_SIGN_IDENTITY (SHA-1 of the identity) and MACOS_KEYCHAIN, appended to # $GITHUB_ENV when running in Actions, otherwise printed. # @@ -20,15 +22,22 @@ fi TMP="${RUNNER_TEMP:-${TMPDIR:-/tmp}}" KEYCHAIN="$TMP/pdfcraft-signing.keychain-db" CERT="$TMP/pdfcraft-signing.p12" +INSTALLER_CERT="$TMP/pdfcraft-installer-signing.p12" KC_PASS="${KEYCHAIN_PASSWORD:-$(openssl rand -hex 24)}" -trap 'rm -f "$CERT"' EXIT +trap 'rm -f "$CERT" "$INSTALLER_CERT"' EXIT printf '%s' "$APPLE_CERTIFICATE" | base64 --decode >"$CERT" security create-keychain -p "$KC_PASS" "$KEYCHAIN" security set-keychain-settings -lut 21600 "$KEYCHAIN" security unlock-keychain -p "$KC_PASS" "$KEYCHAIN" security import "$CERT" -P "${APPLE_CERTIFICATE_PASSWORD:-}" -T /usr/bin/codesign -T /usr/bin/security -t cert -f pkcs12 -k "$KEYCHAIN" rm -f "$CERT" +if [ -n "${APPLE_INSTALLER_CERTIFICATE:-}" ]; then + printf '%s' "$APPLE_INSTALLER_CERTIFICATE" | base64 --decode >"$INSTALLER_CERT" + security import "$INSTALLER_CERT" -P "${APPLE_INSTALLER_CERTIFICATE_PASSWORD:-}" \ + -T /usr/bin/productbuild -T /usr/bin/productsign -T /usr/bin/security -t cert -f pkcs12 -k "$KEYCHAIN" + rm -f "$INSTALLER_CERT" +fi # Let codesign use the key without a UI prompt. security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KC_PASS" "$KEYCHAIN" >/dev/null # Put it on the search list (keeping the login keychain) so codesign and notarytool find it. @@ -42,9 +51,15 @@ if [ -z "$IDENTITY" ]; then exit 1 fi echo "Imported signing identity $IDENTITY" +INSTALLER_IDENTITY="$(security find-identity -v "$KEYCHAIN" | awk '/Developer ID Installer/ { print $2; exit }')" +if [ -n "${APPLE_INSTALLER_CERTIFICATE:-}" ] && [ -z "$INSTALLER_IDENTITY" ]; then + echo "error: no 'Developer ID Installer' identity in APPLE_INSTALLER_CERTIFICATE" >&2 + exit 1 +fi if [ -n "${GITHUB_ENV:-}" ]; then - { echo "MACOS_SIGN_IDENTITY=$IDENTITY"; echo "MACOS_KEYCHAIN=$KEYCHAIN"; } >>"$GITHUB_ENV" + { echo "MACOS_SIGN_IDENTITY=$IDENTITY"; echo "MACOS_KEYCHAIN=$KEYCHAIN"; echo "MACOS_INSTALLER_SIGN_IDENTITY=$INSTALLER_IDENTITY"; } >>"$GITHUB_ENV" else echo "MACOS_SIGN_IDENTITY=$IDENTITY" echo "MACOS_KEYCHAIN=$KEYCHAIN" + echo "MACOS_INSTALLER_SIGN_IDENTITY=$INSTALLER_IDENTITY" fi diff --git a/packaging/macos/package.sh b/packaging/macos/package.sh index e04be13c8d..17a2b9ff3a 100755 --- a/packaging/macos/package.sh +++ b/packaging/macos/package.sh @@ -40,7 +40,9 @@ IDENTITY="${MACOS_SIGN_IDENTITY:--}" SHORT_VERSION="${VERSION%%-*}" WORK="$CARGO_TARGET_DIR/macos-package" APP="$WORK/PdfCraft.app" +HOT_APP="$WORK/PdfCraft Hot Folder.app" DMG="$DIST/pdfcraft-$VERSION-macos-$ARCH.dmg" +PKG="$DIST/pdfcraft-$VERSION-macos-$ARCH.pkg" CLI_ZIP="$DIST/pdfcraft-cli-$VERSION-macos-$ARCH.zip" NOTARIZE=0 @@ -58,12 +60,12 @@ echo "==> PdfCraft $VERSION for macOS ($ARCH), identity: $IDENTITY, notarize: $N if [ "$SKIP_BUILD" = 0 ]; then args=() for t in "${TARGETS[@]}"; do args+=(--target "$t"); done - (cd "$ROOT" && cargo build --release --locked -p pdfcraft -p pdfcraft-cli "${args[@]}") + (cd "$ROOT" && cargo build --release --locked -p pdfcraft -p pdfcraft-cli -p pdfcraft-hot-folder "${args[@]}") fi rm -rf "$WORK" mkdir -p "$WORK/bin" -for bin in pdfcraft pdfcraft-cli; do +for bin in pdfcraft pdfcraft-cli pdfcraft-hot-folder; do inputs=() for t in "${TARGETS[@]}"; do inputs+=("$CARGO_TARGET_DIR/$t/release/$bin"); done lipo -create -output "$WORK/bin/$bin" "${inputs[@]}" @@ -127,6 +129,52 @@ if [ "$NOTARIZE" = 1 ]; then spctl --assess --type execute -vvv "$APP" fi +# ---- optional Hot Folder app and component installer ------------------------------------------- +mkdir -p "$HOT_APP/Contents/MacOS" "$HOT_APP/Contents/Resources" +cp "$WORK/bin/pdfcraft-hot-folder" "$HOT_APP/Contents/MacOS/PdfCraft Hot Folder" +cp "$ROOT/assets/app-icon/pdfcraft.icns" "$HOT_APP/Contents/Resources/PdfCraft.icns" +copy_font_licences "$HOT_APP/Contents/Resources" +stage_models "$HOT_APP/Contents/Resources/models" +sed -e "s/@VERSION@/$VERSION/g" -e "s/@SHORT_VERSION@/$SHORT_VERSION/g" \ + -e "s/@BUILD_SHA@/${PDFCRAFT_BUILD_SHA:-unknown}/g" \ + "$HERE/HotFolder-Info.plist.in" >"$HOT_APP/Contents/Info.plist" +plutil -lint "$HOT_APP/Contents/Info.plist" +sign --options runtime --entitlements "$HERE/entitlements.plist" "$HOT_APP/Contents/MacOS/PdfCraft Hot Folder" +sign --options runtime --entitlements "$HERE/entitlements.plist" "$HOT_APP" +codesign --verify --strict --deep --verbose=2 "$HOT_APP" +if [ "$NOTARIZE" = 1 ]; then + ditto -c -k --keepParent "$HOT_APP" "$WORK/HotFolder-notarize.zip" + notarize "$WORK/HotFolder-notarize.zip" + xcrun stapler staple "$HOT_APP" +fi + +# The native Installer shows a default-off companion choice. The regular DMG stays a simple +# drag-to-Applications installation; system services are registered only by the selected PKG. +mkdir -p "$WORK/components" "$WORK/hot-folder-scripts" +cp "$HERE/hot-folder-scripts/preinstall" "$HERE/hot-folder-scripts/postinstall" "$WORK/hot-folder-scripts/" +chmod 755 "$WORK/hot-folder-scripts/"* +pkgbuild --component "$APP" --install-location /Applications --identifier ai.storyteller.pdfcraft.pkg \ + --version "$SHORT_VERSION" "$WORK/components/PdfCraft-component.pkg" +pkgbuild --component "$HOT_APP" --install-location /Applications --identifier ai.storyteller.pdfcraft.hot-folder.pkg \ + --version "$SHORT_VERSION" --scripts "$WORK/hot-folder-scripts" "$WORK/components/HotFolder-component.pkg" +sed "s/@SHORT_VERSION@/$SHORT_VERSION/g" "$HERE/Distribution.xml.in" >"$WORK/Distribution.xml" +installer_signature=() +if [ -n "${MACOS_INSTALLER_SIGN_IDENTITY:-}" ]; then + installer_signature=(--sign "$MACOS_INSTALLER_SIGN_IDENTITY") + if [ -n "${MACOS_KEYCHAIN:-}" ]; then installer_signature+=(--keychain "$MACOS_KEYCHAIN"); fi +else + warn "PKG has no Developer ID Installer signature (MACOS_INSTALLER_SIGN_IDENTITY is unset)" +fi +productbuild --distribution "$WORK/Distribution.xml" --package-path "$WORK/components" \ + ${installer_signature[@]+"${installer_signature[@]}"} "$PKG" +pkgutil --expand "$PKG" "$WORK/pkg-check" +test -f "$WORK/pkg-check/Distribution" +if [ "$NOTARIZE" = 1 ] && [ -n "${MACOS_INSTALLER_SIGN_IDENTITY:-}" ]; then + notarize "$PKG" + xcrun stapler staple "$PKG" + xcrun stapler validate "$PKG" +fi + # ---- DMG --------------------------------------------------------------------------------------- echo "==> building $DMG" STAGE="$WORK/dmg" @@ -193,4 +241,4 @@ if [ "$NOTARIZE" = 1 ]; then notarize "$CLI_ZIP"; fi "$WORK/bin/pdfcraft-cli" --version echo "==> done" -ls -lh "$DMG" "$CLI_ZIP" +ls -lh "$DMG" "$PKG" "$CLI_ZIP" diff --git a/packaging/service-tests/linux-deb.sh b/packaging/service-tests/linux-deb.sh new file mode 100755 index 0000000000..4fbb58e1e5 --- /dev/null +++ b/packaging/service-tests/linux-deb.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Disposable CI machine only: exercise real package scripts and restricted service identity. +set -euo pipefail +work="$RUNNER_TEMP/hot-folder-deb" +mkdir -p "$work/main/DEBIAN" "$work/stage/usr/bin" +cat > "$work/main/DEBIAN/control" <<'EOF' +Package: pdfcraft +Version: 0.6.0 +Architecture: all +Maintainer: PdfCraft contributors +Description: Synthetic main-package dependency for installer integration testing +EOF +dpkg-deb --build "$work/main" "$work/main.deb" +sudo dpkg -i "$work/main.deb" +if systemctl is-enabled pdfcraft-hot-folder.service 2>/dev/null; then + echo 'Installing only the main package unexpectedly enabled Hot Folder' >&2; exit 1 +fi +cp target/debug/examples/service_probe "$work/stage/usr/bin/pdfcraft-hot-folder" +export NFPM_ARCH=amd64 VERSION=0.6.0 PDFCRAFT_MAINTAINER='PdfCraft contributors' +sed "s|\${HOT_STAGE}|$work/stage|g" packaging/linux/hot-folder-nfpm.yaml > "$work/nfpm.yaml" +nfpm package -f "$work/nfpm.yaml" -p deb -t "$work/hot-folder.deb" +sudo dpkg -i "$work/hot-folder.deb" +probe=/usr/bin/pdfcraft-hot-folder +config=/var/lib/pdfcraft-hot-folder/tasks.json +for _attempt in $(seq 1 40); do + sudo test -f /var/lib/pdfcraft-hot-folder/tasks.heartbeat && break + sleep 0.25 +done +sudo test -f /var/lib/pdfcraft-hot-folder/tasks.heartbeat +"$probe" --service-action status --config "$config" | grep 'installed=true enabled=true running=true account=pdfcraft-hot-folder' +sudo runuser -u pdfcraft-hot-folder -- sh -c 'printf "{\"tasks\":[],\"test_marker\":\"preserved\"}" > /var/lib/pdfcraft-hot-folder/tasks.json' +sudo dpkg -i "$work/hot-folder.deb" +sudo grep preserved "$config" +sudo dpkg -r pdfcraft-hot-folder +test ! -e /etc/systemd/system/pdfcraft-hot-folder.service +sudo grep preserved "$config" +sudo dpkg -r pdfcraft diff --git a/packaging/service-tests/macos-pkg.sh b/packaging/service-tests/macos-pkg.sh new file mode 100755 index 0000000000..6ea2987221 --- /dev/null +++ b/packaging/service-tests/macos-pkg.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Disposable CI machine only: test the actual optional Installer choice and postinstall. +set -euo pipefail +work="$RUNNER_TEMP/hot-folder-pkg" +hot="$work/PdfCraft Hot Folder.app" +main="$work/PdfCraft.app" +mkdir -p "$hot/Contents/MacOS" "$main/Contents/MacOS" "$work/components" "$work/scripts" +cp target/debug/examples/service_probe "$hot/Contents/MacOS/PdfCraft Hot Folder" +cp target/debug/examples/service_probe "$main/Contents/MacOS/PdfCraft" +sed -e 's/@SHORT_VERSION@/0.6.0/g' -e 's/@VERSION@/0.6.0/g' -e 's/@BUILD_SHA@/service-test/g' \ + packaging/macos/HotFolder-Info.plist.in > "$hot/Contents/Info.plist" +sed -e 's/PdfCraft Hot Folder/PdfCraft/g' -e 's/ai.storyteller.pdfcraft.hot-folder/ai.storyteller.pdfcraft/g' \ + "$hot/Contents/Info.plist" > "$main/Contents/Info.plist" +cp packaging/macos/hot-folder-scripts/* "$work/scripts/" +chmod 755 "$work/scripts/"* +codesign --force --sign - "$hot" +codesign --force --sign - "$main" +pkgbuild --component "$main" --install-location /Applications --identifier ai.storyteller.pdfcraft.pkg \ + --version 0.6.0 "$work/components/PdfCraft-component.pkg" +pkgbuild --component "$hot" --install-location /Applications --identifier ai.storyteller.pdfcraft.hot-folder.pkg \ + --version 0.6.0 --scripts "$work/scripts" "$work/components/HotFolder-component.pkg" +sed 's/@SHORT_VERSION@/0.6.0/g' packaging/macos/Distribution.xml.in > "$work/Distribution.xml" +productbuild --distribution "$work/Distribution.xml" --package-path "$work/components" "$work/PdfCraft.pkg" +sudo installer -pkg "$work/PdfCraft.pkg" -target / +test -d /Applications/PdfCraft.app +test ! -d '/Applications/PdfCraft Hot Folder.app' +cat > "$work/choices.plist" <<'EOF' + +choiceIdentifierhot-folderchoiceAttributeselectedattributeSetting1 +EOF +sudo installer -pkg "$work/PdfCraft.pkg" -target / -applyChoiceChangesXML "$work/choices.plist" +probe='/Applications/PdfCraft Hot Folder.app/Contents/MacOS/PdfCraft Hot Folder' +config="$HOME/Library/Application Support/PdfCraft/HotFolder/tasks.json" +for _attempt in $(seq 1 40); do + [ -f "${config%.json}.heartbeat" ] && break + sleep 0.25 +done +test -f "${config%.json}.heartbeat" +"$probe" --service-action status --config "$config" | grep "installed=true enabled=true running=true account=$(id -un)" +# Updating the selected component must keep the existing task file and restart the daemon. +printf '{"tasks":[],"test_marker":"preserved"}' > "$config" +sudo installer -pkg "$work/PdfCraft.pkg" -target / -applyChoiceChangesXML "$work/choices.plist" +grep preserved "$config" +sudo "$probe" --service-action uninstall --config "$config" +"$probe" --service-action status --config "$config" | grep 'installed=false' +grep preserved "$config" diff --git a/packaging/service-tests/unix.sh b/packaging/service-tests/unix.sh new file mode 100755 index 0000000000..d99cade0d8 --- /dev/null +++ b/packaging/service-tests/unix.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Runs only on disposable CI machines. Verifies real boot-service registration and termination. +set -euo pipefail +service_dir=/Library/PdfCraftServiceTest +if [ "$(uname -s)" = Linux ]; then service_dir=/usr/lib/pdfcraft-service-test; fi +sudo install -d -m 755 "$service_dir" +sudo install -m 755 target/debug/examples/service_probe "$service_dir/pdfcraft-hot-folder" +probe="$service_dir/pdfcraft-hot-folder" +directory="$RUNNER_TEMP/Hot Folder service test" +mkdir -m 700 "$directory" +config="$directory/tasks.json" +(umask 077; printf '{}\n' > "$config") +# The diagnostic log intentionally belongs to the unprivileged CI account. +# shellcheck disable=SC2024 +if sudo "$probe" --service --config "$config" > "$directory/root-run.log" 2>&1; then + echo 'The processing worker must reject root execution' >&2; exit 1 +fi +grep 'non-root account' "$directory/root-run.log" +test ! -e "$directory/tasks.heartbeat" +cleanup() { sudo "$probe" --service-action uninstall --config "$config" || true; } +trap cleanup EXIT +sudo "$probe" --service-action install --account "$(id -un)" --config "$config" +for _attempt in $(seq 1 40); do + [ -f "$directory/tasks.heartbeat" ] && break + sleep 0.25 +done +test -f "$directory/tasks.heartbeat" +"$probe" --service-action status --config "$config" | grep 'installed=true enabled=true running=true' +sudo "$probe" --service-action disable --config "$config" | grep 'enabled=false' +sudo "$probe" --service-action enable --config "$config" | grep 'enabled=true' +sudo "$probe" --service-action stop --config "$config" +for _attempt in $(seq 1 40); do + [ -f "$directory/tasks.stopped" ] && break + sleep 0.25 +done +test -f "$directory/tasks.stopped" +"$probe" --service-action status --config "$config" | grep 'running=false' +sudo "$probe" --service-action start --config "$config" +sudo "$probe" --service-action uninstall --config "$config" +"$probe" --service-action status --config "$config" | grep 'installed=false' +trap - EXIT diff --git a/packaging/service-tests/windows-msi.ps1 b/packaging/service-tests/windows-msi.ps1 new file mode 100644 index 0000000000..80545d25fe --- /dev/null +++ b/packaging/service-tests/windows-msi.ps1 @@ -0,0 +1,57 @@ +# Disposable CI machine only: build the real installer definition around a synthetic +# service executable, then test opt-in installation and component removal. +$ErrorActionPreference = 'Stop' +$stage = Join-Path $env:RUNNER_TEMP 'hot-folder-msi' +New-Item -ItemType Directory -Path "$stage/models" -Force | Out-Null +foreach ($name in 'pdfcraft.exe', 'pdfcraft-cli.exe', 'pdfcraft-hot-folder.exe') { + Copy-Item -LiteralPath 'target/debug/examples/service_probe.exe' -Destination (Join-Path $stage $name) +} +foreach ($name in 'LICENSE-MIT', 'LICENSE-APACHE', 'NOTICE') { Copy-Item -LiteralPath $name -Destination $stage } +Copy-Item -LiteralPath 'assets/app-icon/LICENSE-document.txt' -Destination $stage +Set-Content -LiteralPath "$stage/models/detection.rten" -Value 'synthetic installer fixture; never an OCR model' +Set-Content -LiteralPath "$stage/models/recognition.rten" -Value 'synthetic installer fixture; never an OCR model' +$msi = Join-Path $stage 'service-test.msi' +$upgrade = Join-Path $stage 'service-test-upgrade.msi' +$installedMsi = $msi +& wix extension add -g WixToolset.Util.wixext/5.0.2 +if ($LASTEXITCODE -ne 0) { throw 'WiX extension installation failed' } +& wix build packaging/windows/pdfcraft.wxs packaging/windows/installer-ui.wxs -arch x64 -ext WixToolset.Util.wixext -d Version=0.6.0 -d "BinDir=$stage" -d "ModelsDir=$stage/models" -d 'IconPath=assets/app-icon/pdfcraft.ico' -o $msi +if ($LASTEXITCODE -ne 0) { throw 'WiX compilation failed' } +& pwsh -NoProfile -File packaging/windows/test-msi.ps1 $msi +if ($LASTEXITCODE -ne 0) { throw 'Compiled MSI validation failed' } +& wix build packaging/windows/pdfcraft.wxs packaging/windows/installer-ui.wxs -arch x64 -ext WixToolset.Util.wixext -d Version=0.6.1 -d "BinDir=$stage" -d "ModelsDir=$stage/models" -d 'IconPath=assets/app-icon/pdfcraft.ico' -o $upgrade +if ($LASTEXITCODE -ne 0) { throw 'Upgrade MSI compilation failed' } +function Invoke-Msi([string] $Options, [string] $Name) { + $log = Join-Path $stage "$Name.log" + $process = Start-Process -FilePath msiexec.exe -ArgumentList "$Options /qn /norestart /l*v `"$log`"" -Wait -PassThru -WindowStyle Hidden + if ($process.ExitCode -notin @(0,3010)) { + Select-String -LiteralPath $log -Pattern 'Return value 3|Error [0-9]|failed' -Context 12,4 + throw "MSI $Name failed: $($process.ExitCode)" + } +} +$data = Join-Path $env:ProgramData 'PdfCraft/HotFolder' +try { + Invoke-Msi "/i `"$msi`"" 'default' + if (Get-Service PdfCraftHotFolder -ErrorAction SilentlyContinue) { throw 'Default install unexpectedly installed the service' } + Invoke-Msi "/i `"$msi`" ADDLOCAL=HotFolder INSTALLHOTFOLDER=1" 'opt-in' + $service = Get-CimInstance Win32_Service -Filter "Name='PdfCraftHotFolder'" + if ($service.State -ne 'Running' -or $service.StartMode -ne 'Auto' -or $service.StartName -ne 'NT AUTHORITY\LocalService') { + throw 'Installed service is not running automatically as LocalService' + } + for ($i = 0; $i -lt 40 -and -not (Test-Path -LiteralPath "$data/tasks.heartbeat"); $i++) { Start-Sleep -Milliseconds 250 } + if (-not (Test-Path -LiteralPath "$data/tasks.heartbeat")) { throw 'Installed service cannot write its protected task directory' } + Set-Content -LiteralPath "$data/keep-user-tasks.txt" -Value 'synthetic task preservation fixture' + # No INSTALLHOTFOLDER override: a major upgrade must preserve the opt-in choice. + Invoke-Msi "/i `"$upgrade`"" 'upgrade' + $installedMsi = $upgrade + $service = Get-CimInstance Win32_Service -Filter "Name='PdfCraftHotFolder'" + if ($service.State -ne 'Running' -or $service.StartMode -ne 'Auto' -or $service.StartName -ne 'NT AUTHORITY\LocalService') { + throw 'Upgrade did not preserve the optional automatic service and its account' + } + if (-not (Test-Path -LiteralPath "$data/keep-user-tasks.txt")) { throw 'Upgrade deleted task data' } + Invoke-Msi "/i `"$installedMsi`" REMOVE=HotFolder" 'remove-companion' + if (Get-Service PdfCraftHotFolder -ErrorAction SilentlyContinue) { throw 'Removing the component left its service installed' } + if (-not (Test-Path -LiteralPath "$data/keep-user-tasks.txt")) { throw 'Removing the component deleted task data' } +} finally { + Invoke-Msi "/x `"$installedMsi`"" 'uninstall' +} diff --git a/packaging/service-tests/windows.ps1 b/packaging/service-tests/windows.ps1 new file mode 100644 index 0000000000..7ace401d5d --- /dev/null +++ b/packaging/service-tests/windows.ps1 @@ -0,0 +1,34 @@ +# Disposable CI machine only. Does not require a logged-in desktop for the worker. +$ErrorActionPreference = 'Stop' +$probe = (Resolve-Path -LiteralPath 'target/debug/examples/service_probe.exe').Path +$directory = Join-Path $env:RUNNER_TEMP 'Hot Folder service test' +New-Item -ItemType Directory -Path $directory | Out-Null +$config = Join-Path $directory 'tasks.json' +Set-Content -LiteralPath $config -Value '{}' +& icacls.exe $directory /grant '*S-1-5-19:(OI)(CI)F' | Out-Null +if ($LASTEXITCODE -ne 0) { throw 'Could not grant LocalService access to the synthetic test directory' } +$command = '"' + $probe + '" --service --config "' + $config + '"' +& sc.exe create PdfCraftHotFolder binPath= $command start= auto obj= 'NT AUTHORITY\LocalService' +if ($LASTEXITCODE -ne 0) { throw 'Service registration failed' } +function Invoke-Probe([string] $Action) { + $result = & $probe --service-action $Action --config $config + if ($LASTEXITCODE -ne 0) { throw "Service probe failed: $Action" } + return $result +} +try { + Invoke-Probe start + for ($i = 0; $i -lt 40 -and -not (Test-Path -LiteralPath (Join-Path $directory 'tasks.heartbeat')); $i++) { Start-Sleep -Milliseconds 250 } + if (-not (Test-Path -LiteralPath (Join-Path $directory 'tasks.heartbeat'))) { throw 'Service never processed its synthetic configuration' } + if ((Invoke-Probe status) -notmatch 'installed=true enabled=true running=true') { throw 'Wrong running/startup state' } + $service = Get-CimInstance Win32_Service -Filter "Name='PdfCraftHotFolder'" + if ($service.StartName -ne 'NT AUTHORITY\LocalService') { throw 'Service is not running as LocalService' } + if ((Invoke-Probe disable) -notmatch 'enabled=false') { throw 'Disabling boot startup failed' } + if ((Invoke-Probe enable) -notmatch 'enabled=true') { throw 'Enabling boot startup failed' } + Invoke-Probe stop + for ($i = 0; $i -lt 40 -and -not (Test-Path -LiteralPath (Join-Path $directory 'tasks.stopped')); $i++) { Start-Sleep -Milliseconds 250 } + if (-not (Test-Path -LiteralPath (Join-Path $directory 'tasks.stopped'))) { throw 'Service did not terminate gracefully' } + if ((Invoke-Probe status) -notmatch 'running=false') { throw 'Service is still running' } +} finally { + & sc.exe stop PdfCraftHotFolder | Out-Null + & sc.exe delete PdfCraftHotFolder | Out-Null +} diff --git a/packaging/windows/installer-ui.wxs b/packaging/windows/installer-ui.wxs index da96587459..63b4436403 100644 --- a/packaging/windows/installer-ui.wxs +++ b/packaging/windows/installer-ui.wxs @@ -8,34 +8,52 @@ - + - - + + + + + + + + - + - + - + + + + + + + + + - + - + diff --git a/packaging/windows/package.ps1 b/packaging/windows/package.ps1 index 524280b971..bf32133b21 100644 --- a/packaging/windows/package.ps1 +++ b/packaging/windows/package.ps1 @@ -32,6 +32,25 @@ function Invoke-Native([string] $What, [scriptblock] $Block) { if ($LASTEXITCODE -ne 0) { throw "$What failed with exit code $LASTEXITCODE" } } +function Reset-PackageDirectory([string] $Path, [string] $AllowedRoot) { + $rootPath = [IO.Path]::GetFullPath($AllowedRoot).TrimEnd('\') + $targetPath = [IO.Path]::GetFullPath($Path) + if (-not $targetPath.StartsWith($rootPath + '\', [StringComparison]::OrdinalIgnoreCase)) { + throw "Package staging directory is outside the target directory: $targetPath" + } + $current = $targetPath + while ($current -and $current.Length -ge $rootPath.Length) { + if (Test-Path -LiteralPath $current) { + if ((Get-Item -LiteralPath $current -Force).Attributes -band [IO.FileAttributes]::ReparsePoint) { + throw "Package staging cannot traverse a junction or symbolic link: $current" + } + } + $current = [IO.Path]::GetDirectoryName($current) + } + if (Test-Path -LiteralPath $targetPath) { Remove-Item -LiteralPath $targetPath -Recurse -Force } + New-Item -ItemType Directory -Force -Path $targetPath | Out-Null +} + # The version lives in one place: [workspace.package] version in the root Cargo.toml. $Version = $env:PDFCRAFT_VERSION if (-not $Version) { @@ -62,7 +81,7 @@ if (-not $SkipBuild) { [Environment]::SetEnvironmentVariable($flagVar, '-C target-feature=+crt-static') # Fail the build (rather than warn) if the icon/VERSIONINFO can't be embedded. $env:PDFCRAFT_REQUIRE_WINRES = '1' - Invoke-Native "cargo build ($Target)" { cargo build --release --locked -p pdfcraft -p pdfcraft-cli --target $Target } + Invoke-Native "cargo build ($Target)" { cargo build --release --locked -p pdfcraft -p pdfcraft-cli -p pdfcraft-hot-folder --target $Target } } $Bin = Join-Path $TargetDir "$Target\release" @@ -77,16 +96,15 @@ function Get-PeHeader([string] $Path) { return @{ Machine = [BitConverter]::ToUInt16($bytes, $pe + 4); Subsystem = [BitConverter]::ToUInt16($bytes, $pe + 0x5C) } } $Machine = switch ($Arch) { 'x64' { 0x8664 } 'x86' { 0x14C } 'arm64' { 0xAA64 } } -foreach ($check in @(@('pdfcraft.exe', 2), @('pdfcraft-cli.exe', 3))) { +foreach ($check in @(@('pdfcraft.exe', 2), @('pdfcraft-cli.exe', 3), @('pdfcraft-hot-folder.exe', 2))) { $h = Get-PeHeader (Join-Path $Bin $check[0]) if ($h.Machine -ne $Machine) { throw "$($check[0]) is for machine 0x$('{0:X}' -f $h.Machine), expected 0x$('{0:X}' -f $Machine) ($Arch)" } if ($h.Subsystem -ne $check[1]) { throw "$($check[0]) has PE subsystem $($h.Subsystem), expected $($check[1])" } Write-Output "ok $($check[0]): $Arch, PE subsystem $($h.Subsystem)" } $Stage = Join-Path $TargetDir "windows-package\$Arch" -Remove-Item -Recurse -Force $Stage -ErrorAction SilentlyContinue -New-Item -ItemType Directory -Force -Path $Stage | Out-Null -Copy-Item (Join-Path $Bin 'pdfcraft.exe'), (Join-Path $Bin 'pdfcraft-cli.exe') $Stage +Reset-PackageDirectory $Stage $TargetDir +Copy-Item (Join-Path $Bin 'pdfcraft.exe'), (Join-Path $Bin 'pdfcraft-cli.exe'), (Join-Path $Bin 'pdfcraft-hot-folder.exe') $Stage # The document icon embeds Lucide artwork: carry its ISC notice and the project licences # beside the binaries in both distributions. foreach ($f in 'LICENSE-MIT', 'LICENSE-APACHE', 'NOTICE') { @@ -94,7 +112,7 @@ foreach ($f in 'LICENSE-MIT', 'LICENSE-APACHE', 'NOTICE') { } Copy-Item (Join-Path $Root 'assets\app-icon\LICENSE-document.txt') $Stage -& (Join-Path $PSScriptRoot 'sign.ps1') (Join-Path $Stage 'pdfcraft.exe') (Join-Path $Stage 'pdfcraft-cli.exe') +& (Join-Path $PSScriptRoot 'sign.ps1') (Join-Path $Stage 'pdfcraft.exe') (Join-Path $Stage 'pdfcraft-cli.exe') (Join-Path $Stage 'pdfcraft-hot-folder.exe') # OCR models (#103): the app looks in models\ beside pdfcraft.exe. `cargo xtask models` fetches every # ATTRIBUTION.toml `kind = "model"` file (verified by SHA-256) with its licence and ATTRIBUTION.txt; @@ -110,8 +128,9 @@ if (-not (Test-Path (Join-Path $Models 'ATTRIBUTION.txt')) -or -not (Get-ChildIt # ---- MSI --------------------------------------------------------------------------------------- $Msi = Join-Path $Dist "pdfcraft-$Version-windows-$Arch.msi" +Invoke-Native 'WiX service configuration extension' { wix extension add -g WixToolset.Util.wixext/5.0.2 } Invoke-Native 'wix build' { - wix build (Join-Path $PSScriptRoot 'pdfcraft.wxs') -arch $Arch ` + wix build (Join-Path $PSScriptRoot 'pdfcraft.wxs') -arch $Arch -ext WixToolset.Util.wixext ` (Join-Path $PSScriptRoot 'installer-ui.wxs') ` -d "Version=$MsiVersion" -d "BinDir=$Stage" -d "ModelsDir=$Models" -d "IconPath=$(Join-Path $Root 'assets\app-icon\pdfcraft.ico')" ` -o $Msi @@ -125,8 +144,7 @@ Remove-Item -Force -ErrorAction SilentlyContinue ([IO.Path]::ChangeExtension($Ms # ---- portable zip ------------------------------------------------------------------------------ $Portable = Join-Path $TargetDir "windows-package\pdfcraft-$Version-windows-$Arch-portable" -Remove-Item -Recurse -Force $Portable -ErrorAction SilentlyContinue -New-Item -ItemType Directory -Force -Path $Portable | Out-Null +Reset-PackageDirectory $Portable $TargetDir Copy-Item (Join-Path $Stage '*.exe') $Portable foreach ($f in 'NOTICE', 'LICENSE-document.txt') { Copy-Item (Join-Path $Stage $f) $Portable diff --git a/packaging/windows/pdfcraft.wxs b/packaging/windows/pdfcraft.wxs index 11d2e39ab0..84b1f3391e 100644 --- a/packaging/windows/pdfcraft.wxs +++ b/packaging/windows/pdfcraft.wxs @@ -9,7 +9,7 @@ and in Default apps, never forced as the default). Files opened with PdfCraft arrive on its command line. --> - + + + + + + + @@ -49,14 +56,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/windows/test-msi.ps1 b/packaging/windows/test-msi.ps1 index 9c0d50b575..947a3f68e2 100644 --- a/packaging/windows/test-msi.ps1 +++ b/packaging/windows/test-msi.ps1 @@ -145,6 +145,35 @@ try { if ($modelFiles -lt 2) { throw "expected the OCR models (*.rten) in models\, found $modelFiles" } $rm = Read-Row 'SELECT `Dialog` FROM `Dialog` WHERE `Dialog` = ''MsiRMFilesInUse''' 1 Assert-Equal $rm[0] 'MsiRMFilesInUse' 'Files-in-use dialog' +# The companion is opt-in and belongs to a separate feature. Its service runs at boot +# under LocalService, never an interactive user's credentials or LocalSystem. +$feature = Read-Row 'SELECT `Level` FROM `Feature` WHERE `Feature` = ''HotFolder''' 1 +Assert-Equal $feature[0] '2' 'Hot Folder default feature level' +Assert-NoRow 'SELECT `Value` FROM `Property` WHERE `Property` = ''INSTALLHOTFOLDER''' 'Hot Folder enabled by default' +if (($secure[0] -split ';') -notcontains 'INSTALLHOTFOLDER') { throw 'Hot Folder choice is not secure' } +$choice = Read-Row 'SELECT `Type`, `Property` FROM `Control` WHERE `Dialog_` = ''InstallWelcome'' AND `Control` = ''HotFolder''' 2 +Assert-Equal $choice[0] 'CheckBox' 'Hot Folder optional checkbox' +Assert-Equal $choice[1] 'INSTALLHOTFOLDER' 'Hot Folder checkbox property' +$service = Read-Row 'SELECT `ServiceType`, `StartType`, `StartName`, `Arguments`, `Component_` FROM `ServiceInstall` WHERE `Name` = ''PdfCraftHotFolder''' 5 +Assert-Equal $service[0] '16' 'Hot Folder own-process service' +Assert-Equal $service[1] '2' 'Hot Folder automatic boot startup' +Assert-Equal $service[2] 'NT AUTHORITY\LocalService' 'Hot Folder restricted service account' +Assert-Equal $service[3] '--service --config "[HotFolderData]tasks.json"' 'Service configuration arguments' +Assert-Equal $service[4] 'PdfcraftHotFolderApp' 'Service executable component' +$control = Read-Row 'SELECT `Event`, `Wait` FROM `ServiceControl` WHERE `Name` = ''PdfCraftHotFolder''' 2 +Assert-Equal $control[0] '163' 'Start on install, stop on upgrade/removal, delete on uninstall' +Assert-Equal $control[1] '1' 'Wait for graceful service stop' +$acl = Read-Row 'SELECT `SDDLText` FROM `MsiLockPermissionsEx` WHERE `LockObject` = ''HotFolderData''' 1 +Assert-Equal $acl[0] 'D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;S-1-5-19)(A;OICI;GRGX;;;BU)' 'Protected task directory' +$secrets = Read-Row 'SELECT `SDDLText` FROM `MsiLockPermissionsEx` WHERE `LockObject` = ''HotFolderSecrets''' 1 +Assert-Equal $secrets[0] 'D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;S-1-5-19)' 'Credentials must not be readable by ordinary users' +$shortcut = Read-Row 'SELECT `Target`, `Arguments` FROM `Shortcut` WHERE `Shortcut` = ''HotFolderStartMenuShortcut''' 2 +Assert-Equal $shortcut[0] '[#PdfcraftHotFolderExe]' 'Hot Folder shortcut executable' +Assert-Equal $shortcut[1] '--config "[HotFolderData]tasks.json"' 'GUI and service use the same tasks' +foreach ($action in @(@('AddHotFolder', 'AddLocal'), @('RemoveHotFolder', 'Remove'))) { + $event = Read-Row ('SELECT `Argument` FROM `ControlEvent` WHERE `Dialog_` = ''InstallMaintenance'' AND `Control_` = ''' + $action[0] + ''' AND `Event` = ''' + $action[1] + '''') 1 + Assert-Equal $event[0] 'HotFolder' "$($action[0]) feature action" +} [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($Database) [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($Installer) Write-Output 'ok MSI: per-machine scope guard, publisher, persistent progress text, Start Menu shortcut, optional desktop shortcut (default on, checkbox), icon/key path, full-UI success/cancel/error and Finish controls, files-in-use dialog, OCR models' diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index b9e120f4d4..97e0343280 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -10060,3 +10060,15 @@ commands = [] tools = [] evidence = [] notes = "" + +[[feature]] +id = "misc.hot-folder" +title = "Standalone watched-folder document processing" +area = "N" +tier = "P1" +milestone = "M13" +status = "partial" +commands = [] +tools = ["hot_folder_configure", "hot_folder_status", "hot_folder_poll", "hot_folder_retry", "hot_folder_run_now", "hot_folder_source", "hot_folder_project", "hot_folder_service"] +evidence = ["crates/engine/src/hot_folder.rs::hot_folder_settling_and_durable_receipts_prevent_duplicates_and_overwrites", "crates/automation/tests/hot_folder.rs::hot_folder_tools_produce_searchable_output_and_preserve_source", "apps/pdfcraft-hot-folder/src/ui.rs::hot_folder_ui_has_seven_languages_and_persists_a_stopped_task"] +notes = "Separate native application: local/network/FTP/Outlook/open-project sources, PDF/images, durable receipts, calendar scheduling, OCR and scan preprocessing, JSON analysis without OCR, grouped PDF/Office/XLSX/EPUB/image outputs, safe source policies, task import/export and seven UI languages. Boot services use restricted accounts. Full proprietary project/output parity remains partial; see docs/hot-folder.md. No implicit service or MCP startup." diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 72a199c10d..3db0413739 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -77,6 +77,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("engine", Class::Layer(6)), // L7 platform + frontends ("platform", Class::Layer(7)), + ("hot-folder-service", Class::Layer(7)), ("ui-common", Class::Layer(7)), ("ui-egui", Class::Layer(7)), ("automation", Class::Layer(7)), @@ -86,6 +87,7 @@ pub const TABLE: &[(&str, Class)] = &[ // L8 apps + tooling ("pdfcraft", Class::Exempt), ("cli", Class::Exempt), + ("hot-folder", Class::Exempt), ("web", Class::Exempt), ("xtask", Class::Exempt), ]; @@ -95,6 +97,7 @@ pub const STANDALONE_DEPS: &[(&str, &[&str])] = &[("cos", &["filters", "crypt", /// Allowed same-layer edges (architecture §3 rule 1). `forms → js` is deliberately absent. pub const SIDEWAYS: &[(&str, &str)] = &[ + ("automation", "hot-folder-service"), ("redact", "edit"), ("a11y", "preflight"), ("prepress", "preflight"), From 486653b2a7b5c286758bae802410579aa29bbeb9 Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:27:50 +0200 Subject: [PATCH 05/19] M10: version Hot Folder service path dependencies Satisfy the existing cargo-deny wildcard policy without changing dependency resolution. Full dependency audit and workspace tests pass. Co-authored-by: Codex --- apps/pdfcraft-hot-folder/Cargo.toml | 2 +- crates/automation/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/pdfcraft-hot-folder/Cargo.toml b/apps/pdfcraft-hot-folder/Cargo.toml index 5a648a93e0..0949c8acb3 100644 --- a/apps/pdfcraft-hot-folder/Cargo.toml +++ b/apps/pdfcraft-hot-folder/Cargo.toml @@ -11,7 +11,7 @@ workspace = true [dependencies] pdfcraft-engine = { workspace = true } pdfcraft-fonts = { workspace = true } -pdfcraft-hot-folder-service = { path = "../../crates/hot-folder-service" } +pdfcraft-hot-folder-service = { path = "../../crates/hot-folder-service", version = "0.6.0" } eframe = { workspace = true, features = ["default_fonts", "glow", "accesskit", "wayland", "x11"] } egui = { workspace = true } rfd = { workspace = true } diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index c554d69d66..522d61959d 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -24,7 +24,7 @@ default = ["mcp"] mcp = ["dep:base64"] [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -pdfcraft-hot-folder-service = { path = "../hot-folder-service" } +pdfcraft-hot-folder-service = { path = "../hot-folder-service", version = "0.6.0" } zeroize = "1.8" [dev-dependencies] From 26b989caae436ac93a9a55372c47ac1c01d466f9 Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:37:48 +0200 Subject: [PATCH 06/19] Harden Hot Folder supervision, durable recovery and Windows service upgrades --- .github/actionlint.yaml | 1 + .github/workflows/ci.yml | 2 + .github/workflows/hot-folder-server.yml | 142 ++++++ .github/workflows/hot-folder-services.yml | 15 +- .github/workflows/packaging-lint.yml | 1 + Cargo.lock | 275 ++++++++-- ROADMAP.md | 2 + apps/pdfcraft-hot-folder/Cargo.toml | 8 + apps/pdfcraft-hot-folder/src/main.rs | 30 +- apps/pdfcraft-hot-folder/src/worker.rs | 234 ++++++++- .../pdfcraft-hot-folder/src/worker_process.rs | 476 +++++++++++++++++ apps/pdfcraft-hot-folder/tests/worker.rs | 95 ++++ crates/automation/README.md | 10 + crates/automation/src/hot_folder_service.rs | 5 +- crates/automation/src/tools.rs | 4 +- crates/automation/tests/hostile_args.rs | 42 +- crates/engine/Cargo.toml | 2 + crates/engine/src/hot_folder.rs | 480 ++++++++++++++---- crates/engine/src/hot_folder_journal.rs | 184 +++++++ crates/engine/src/hot_folder_output.rs | 60 ++- crates/engine/src/hot_folder_preprocess.rs | 37 +- crates/engine/src/hot_folder_server_tests.rs | 374 ++++++++++++++ crates/engine/src/hot_folder_sources.rs | 158 +++++- .../src/hot_folder_sources/credentials.rs | 46 +- .../engine/src/hot_folder_sources/project.rs | 10 +- crates/engine/src/lib.rs | 14 + .../examples/service_probe.rs | 5 + crates/hot-folder-service/src/lib.rs | 12 + crates/hot-folder-service/src/windows.rs | 25 +- .../tests/{updates.rs => release_checks.rs} | 0 docs/hot-folder-installation.md | 74 ++- docs/hot-folder.md | 37 +- docs/roadmap.md | 2 +- packaging/server-tests/README.txt | 111 ++++ packaging/server-tests/acceptance.ps1 | 358 +++++++++++++ packaging/server-tests/actionlint.yaml | 3 + packaging/server-tests/check.ps1 | 64 +++ packaging/server-tests/common.ps1 | 166 ++++++ packaging/server-tests/verify-provenance.ps1 | 30 ++ packaging/service-tests/windows-msi.ps1 | 46 +- packaging/windows/pdfcraft.wxs | 23 +- packaging/windows/test-msi.ps1 | 12 +- parity/acrobat-features.toml | 2 +- 43 files changed, 3431 insertions(+), 246 deletions(-) create mode 100644 .github/workflows/hot-folder-server.yml create mode 100644 apps/pdfcraft-hot-folder/src/worker_process.rs create mode 100644 apps/pdfcraft-hot-folder/tests/worker.rs create mode 100644 crates/engine/src/hot_folder_journal.rs create mode 100644 crates/engine/src/hot_folder_server_tests.rs rename crates/ui-egui/tests/{updates.rs => release_checks.rs} (100%) create mode 100644 packaging/server-tests/README.txt create mode 100644 packaging/server-tests/acceptance.ps1 create mode 100644 packaging/server-tests/actionlint.yaml create mode 100644 packaging/server-tests/check.ps1 create mode 100644 packaging/server-tests/common.ps1 create mode 100644 packaging/server-tests/verify-provenance.ps1 diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 8c16b6b00a..55b266fb24 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,3 +2,4 @@ self-hosted-runner: labels: - macos-release + - pdfcraft-server-2019-disposable diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c10a3bb32b..4ff7f052d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,8 @@ jobs: - name: Test if: runner.os != 'Linux' + env: + PDFCRAFT_REQUIRE_SYMLINK_TESTS: "1" run: cargo test --workspace - name: Test (with craft-fonts) diff --git a/.github/workflows/hot-folder-server.yml b/.github/workflows/hot-folder-server.yml new file mode 100644 index 0000000000..260b1cf854 --- /dev/null +++ b/.github/workflows/hot-folder-server.yml @@ -0,0 +1,142 @@ +name: Hot Folder Windows Server acceptance + +# Installing services and creating an SMB share belongs on disposable hosts only. +# In particular, no pull_request/pull_request_target or workflow_run trigger may +# route unreviewed code to the opt-in self-hosted Server 2019 runner. +on: + # The existing installer workflow can call the reviewed branch before this + # new file reaches the default branch. Its parent event must still be manual. + workflow_call: + inputs: + installer_run_id: + required: true + type: string + expected_sha: + required: true + type: string + artifact_name: + default: pdfcraft-preview-windows-x64 + type: string + include_server_2019: + default: false + type: boolean + server_2019_confirmation: + default: '' + type: string + workflow_dispatch: + inputs: + installer_run_id: + description: Successful installer workflow run from this repository + required: true + type: string + expected_sha: + description: Full 40-character source SHA (must equal this workflow ref) + required: true + type: string + artifact_name: + description: x64 MSI artifact name in that run + required: true + default: pdfcraft-preview-windows-x64 + type: string + include_server_2019: + description: Also use the dedicated disposable self-hosted Server 2019 runner + default: false + type: boolean + server_2019_confirmation: + description: For 2019 enter exactly DISPOSABLE SERVER 2019 + default: '' + type: string + +permissions: + contents: read + actions: read + +concurrency: + group: hot-folder-server-${{ github.ref }} + cancel-in-progress: false + +jobs: + provenance: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + artifact_id: ${{ steps.verify.outputs.artifact_id }} + steps: + - uses: actions/checkout@v4 + - name: Require matching successful trusted installer run + id: verify + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + INSTALLER_RUN_ID: ${{ inputs.installer_run_id }} + EXPECTED_SHA: ${{ inputs.expected_sha }} + ARTIFACT_NAME: ${{ inputs.artifact_name }} + INCLUDE_2019: ${{ inputs.include_server_2019 }} + CONFIRM_2019: ${{ inputs.server_2019_confirmation }} + run: ./packaging/server-tests/verify-provenance.ps1 + + hosted: + if: github.event_name == 'workflow_dispatch' + needs: provenance + strategy: + fail-fast: false + matrix: + include: + - { os: windows-2022, build: 20348 } + - { os: windows-2025, build: 26100 } + runs-on: ${{ matrix.os }} + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + artifact-ids: ${{ needs.provenance.outputs.artifact_id }} + run-id: ${{ inputs.installer_run_id }} + github-token: ${{ github.token }} + path: installer + merge-multiple: true + - name: Install and exercise the actual OCR service + shell: pwsh + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} + INSTALLER_RUN_ID: ${{ inputs.installer_run_id }} + EXPECTED_BUILD: ${{ matrix.build }} + run: ./packaging/server-tests/acceptance.ps1 -InstallerDirectory "$PWD/installer" -ExpectedCommit $env:EXPECTED_SHA -ExpectedServerBuild $env:EXPECTED_BUILD -DisposableHost + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hot-folder-server-${{ matrix.os }} + path: server-test-results + if-no-files-found: error + + server-2019: + if: github.event_name == 'workflow_dispatch' && inputs.include_server_2019 && inputs.server_2019_confirmation == 'DISPOSABLE SERVER 2019' + needs: provenance + # Configure this environment with required reviewers before enabling the job. + environment: hot-folder-server-2019 + runs-on: [self-hosted, Windows, X64, pdfcraft-server-2019-disposable] + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + artifact-ids: ${{ needs.provenance.outputs.artifact_id }} + run-id: ${{ inputs.installer_run_id }} + github-token: ${{ github.token }} + path: installer + merge-multiple: true + - name: Verify explicitly approved Server 2019 host + shell: pwsh + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} + INSTALLER_RUN_ID: ${{ inputs.installer_run_id }} + run: ./packaging/server-tests/acceptance.ps1 -InstallerDirectory "$PWD/installer" -ExpectedCommit $env:EXPECTED_SHA -ExpectedServerBuild 17763 -DisposableHost + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hot-folder-server-2019 + path: server-test-results + if-no-files-found: error diff --git a/.github/workflows/hot-folder-services.yml b/.github/workflows/hot-folder-services.yml index 0de839d267..06f922f714 100644 --- a/.github/workflows/hot-folder-services.yml +++ b/.github/workflows/hot-folder-services.yml @@ -1,6 +1,15 @@ name: Hot Folder system services on: workflow_dispatch: + push: + branches: [codex/hot-folder] + paths: + - 'crates/hot-folder-service/**' + - 'packaging/service-tests/**' + - 'packaging/windows/**' + - 'packaging/macos/**' + - 'packaging/linux/**' + - '.github/workflows/hot-folder-services.yml' pull_request: paths: - 'crates/hot-folder-service/**' @@ -16,9 +25,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, windows-2025, macos-15] + os: [ubuntu-24.04, windows-2022, windows-2025, macos-15] runs-on: ${{ matrix.os }} - timeout-minutes: 15 + timeout-minutes: 20 steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -48,6 +57,6 @@ jobs: - if: always() && runner.os == 'Windows' uses: actions/upload-artifact@v4 with: - name: hot-folder-msi-lifecycle-logs + name: hot-folder-msi-lifecycle-logs-${{ matrix.os }} path: ${{ runner.temp }}/hot-folder-msi/*.log if-no-files-found: ignore diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index 01277cf8dd..ff6cc078b9 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -44,6 +44,7 @@ jobs: if ($errors) { $errors | ForEach-Object { Write-Output "::error file=$($f.Name)::$_" }; exit 1 } Write-Output "ok $($f.Name)" } + ./packaging/server-tests/check.ps1 - name: XML, desktop entry, AppStream, Flatpak manifest run: | xmllint --noout packaging/windows/*.wxs packaging/macos/*.plist* packaging/linux/*.xml* diff --git a/Cargo.lock b/Cargo.lock index 3e0b6e657c..5ac6066d5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,8 +99,8 @@ dependencies = [ "accesskit_consumer 0.35.0", "hashbrown 0.16.1", "static_assertions", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", ] [[package]] @@ -1022,7 +1022,7 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2242,7 +2242,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ "rustix 1.1.5", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2425,7 +2425,7 @@ dependencies = [ "log", "presser", "thiserror 2.0.21", - "windows", + "windows 0.62.2", ] [[package]] @@ -2667,7 +2667,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -2874,7 +2874,7 @@ dependencies = [ "simd_cesu8", "thiserror 2.0.21", "walkdir", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -3044,7 +3044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -3197,6 +3197,17 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -3213,6 +3224,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "mutants" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ac067452ff1aca8c5002111bd6b1c895baee6e45fcbc44e0193aea17be56" + [[package]] name = "naga" version = "30.0.1" @@ -3331,6 +3348,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "num" version = "0.4.3" @@ -3632,6 +3658,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-io-surface" version = "0.3.2" @@ -3911,7 +3947,7 @@ dependencies = [ "libc", "redox_syscall 0.5.18", "smallvec", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -4113,6 +4149,8 @@ dependencies = [ "ureq", "url", "windows-permissions", + "windows-service", + "winsafe", "zeroize", ] @@ -4169,11 +4207,15 @@ dependencies = [ "pdfcraft-engine", "pdfcraft-fonts", "pdfcraft-hot-folder-service", + "processkit", "rfd", "rustix 1.1.5", + "serde", "serde_json", "sys-locale", + "sysinfo", "tempfile", + "winsafe", "zeroize", ] @@ -4767,6 +4809,23 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "processkit" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1549b59abbe4fc78aa39860e36c81da89262c1278679b8f7299ef4ad7668c004" +dependencies = [ + "async-trait", + "encoding_rs", + "libc", + "mutants", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "tokio-util", + "windows-sys 0.61.2", +] + [[package]] name = "profiling" version = "1.0.18" @@ -5889,6 +5948,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "spirv" version = "0.4.0+sdk-1.4.341.0" @@ -6039,6 +6108,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "sysinfo" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows 0.61.3", +] + [[package]] name = "tag_ptr" version = "0.1.0" @@ -6214,7 +6297,50 @@ version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ + "bytes", + "libc", + "mio", "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", ] [[package]] @@ -6984,9 +7110,9 @@ dependencies = [ "web-sys", "wgpu-naga-bridge", "wgpu-types", - "windows", - "windows-core", - "windows-result", + "windows 0.62.2", + "windows-core 0.62.2", + "windows-result 0.4.1", ] [[package]] @@ -7052,16 +7178,38 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections 0.2.0", + "windows-core 0.61.2", + "windows-future 0.2.1", + "windows-link 0.1.3", + "windows-numerics 0.2.0", +] + [[package]] name = "windows" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-numerics", + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", ] [[package]] @@ -7070,7 +7218,20 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core", + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -7081,9 +7242,20 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading 0.1.0", ] [[package]] @@ -7092,9 +7264,9 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core", - "windows-link", - "windows-threading", + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -7119,20 +7291,36 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + [[package]] name = "windows-numerics" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core", - "windows-link", + "windows-core 0.62.2", + "windows-link 0.2.1", ] [[package]] @@ -7145,13 +7333,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -7165,13 +7362,22 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-strings" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -7207,7 +7413,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -7232,7 +7438,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link", + "windows-link 0.2.1", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -7243,13 +7449,22 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-threading" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index 6a6910acbf..6eefacb515 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -133,6 +133,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- 2026-10-11 — M13: Hot Folder server hardening: isolated bounded task processes, Windows job memory limits, automatic transient retries, durable bounded active journals, crash-safe grouped receipts and deterministic OCR replay. Stable service SID grants preserve custom SCM accounts through repair/upgrade. Added manual real-MSI local/SMB acceptance for Server 2022/2025 and opt-in disposable Server 2019; no untested server certification is claimed. + - 2026-10-11 — M13: optional standalone Hot Folder with durable local/network/FTP/Outlook/open-project processing, calendar schedules, safe source policies, OCR/scan preprocessing, analysis reports, grouped PDF/Office/image/EPUB outputs, headless tools and seven UI languages. Restricted-account boot services and optional Windows/macOS/Linux installers have lifecycle coverage; proprietary ABBYY formats remain incompatible; see [the scope inventory](docs/hot-folder.md). - **2026-10-11 (M10, OCR languages and defaults):** Fold Dutch, Danish, Norwegian Bokmål, Swedish, Finnish and Portuguese into the scan/OCR work, with save/reopen regressions for accented words. The UI selects the primary system/browser language, independently of UI translations, with English fallback; manual selection remains available. Icelandic stays unclaimed. Overall percentage unchanged. diff --git a/apps/pdfcraft-hot-folder/Cargo.toml b/apps/pdfcraft-hot-folder/Cargo.toml index 0949c8acb3..63f9892434 100644 --- a/apps/pdfcraft-hot-folder/Cargo.toml +++ b/apps/pdfcraft-hot-folder/Cargo.toml @@ -16,6 +16,7 @@ eframe = { workspace = true, features = ["default_fonts", "glow", "accesskit", " egui = { workspace = true } rfd = { workspace = true } serde_json = { workspace = true } +serde = { workspace = true } sys-locale = "0.3" chrono = { version = "0.4", default-features = false, features = ["clock"] } tempfile = "3" @@ -24,5 +25,12 @@ zeroize = "1.8" [target.'cfg(unix)'.dependencies] rustix = { version = "1", features = ["process"] } +[target.'cfg(windows)'.dependencies] +processkit = { version = "3.3.4", default-features = false, features = ["limits", "process-control"] } +winsafe = { version = "0.0.29", default-features = false, features = ["kernel"] } + +[target.'cfg(not(windows))'.dependencies] +sysinfo = { version = "0.37", default-features = false, features = ["system"] } + [dev-dependencies] egui_kittest = { workspace = true, features = ["eframe", "wgpu"] } diff --git a/apps/pdfcraft-hot-folder/src/main.rs b/apps/pdfcraft-hot-folder/src/main.rs index ee78cf31e3..c944647030 100644 --- a/apps/pdfcraft-hot-folder/src/main.rs +++ b/apps/pdfcraft-hot-folder/src/main.rs @@ -7,6 +7,7 @@ mod source_ui; mod startup; mod ui; mod worker; +mod worker_process; use pdfcraft_engine::hot_folder as hf; use std::path::{Path, PathBuf}; @@ -51,6 +52,8 @@ fn run() -> Result<(), String> { let mut service_action = None; let mut account = String::new(); let mut initialize = false; + let mut task_child = false; + let mut task_id = None; while let Some(arg) = args.next() { match arg.as_str() { "--config" => path = Some(PathBuf::from(args.next().ok_or("--config needs a file path")?)), @@ -58,8 +61,10 @@ fn run() -> Result<(), String> { "--once" => once = true, "--service" => service = true, "--initialize-config" => initialize = true, + "--task-child" => task_child = true, + "--task-id" => task_id = Some(args.next().ok_or("--task-id needs an ID")?), "--service-action" => { - service_action = Some(args.next().ok_or("--service-action needs status/start/stop/enable/disable/install/uninstall")?) + service_action = Some(args.next().ok_or("--service-action needs status/start/stop/enable/disable/install/uninstall/prepare-account")?) } "--account" => account = args.next().ok_or("--account needs a non-root service account")?, "--version" => { @@ -68,7 +73,7 @@ fn run() -> Result<(), String> { } "--help" | "-h" => { println!( - "PdfCraft Hot Folder\n pdfcraft-hot-folder [--config FILE]\n pdfcraft-hot-folder --worker --config FILE\n pdfcraft-hot-folder --once --config FILE\n pdfcraft-hot-folder --service --config FILE\n pdfcraft-hot-folder --initialize-config --config FILE\n pdfcraft-hot-folder --service-action status|start|stop|enable|disable|install|uninstall --config FILE [--account USER]\nThe GUI saves opt-in tasks. --once runs one scheduler pass; --service hosts a registered boot service. No network listener is opened." + "PdfCraft Hot Folder\n pdfcraft-hot-folder [--config FILE]\n pdfcraft-hot-folder --worker --config FILE\n pdfcraft-hot-folder --once --config FILE\n pdfcraft-hot-folder --service --config FILE\n pdfcraft-hot-folder --initialize-config --config FILE\n pdfcraft-hot-folder --service-action status|start|stop|enable|disable|install|uninstall|prepare-account --config FILE [--account USER]\nThe GUI saves opt-in tasks. --once runs one supervised scheduler pass; --service hosts a registered boot service. Task passes default to a 300-second deadline; local tasks.worker.json can override worker limits. No network listener is opened." ); return Ok(()); } @@ -77,9 +82,22 @@ fn run() -> Result<(), String> { } let config = path.map(Ok).unwrap_or_else(default_config)?; let config = if config.is_absolute() { config } else { std::env::current_dir().map_err(|e| e.to_string())?.join(config) }; - if usize::from(service) + usize::from(once) + usize::from(background) + usize::from(service_action.is_some()) + usize::from(initialize) > 1 { + if usize::from(service) + + usize::from(once) + + usize::from(background) + + usize::from(service_action.is_some()) + + usize::from(initialize) + + usize::from(task_child) + > 1 + { return Err("choose only one operating mode".into()); } + if task_child { + return worker_process::child_main(&config, &task_id.ok_or("the task child needs --task-id")?); + } + if task_id.is_some() { + return Err("--task-id is reserved for supervised task children".into()); + } if initialize { return hf::initialize_config(&config); } @@ -97,15 +115,15 @@ fn run() -> Result<(), String> { "disable" => svc::set_enabled(false, &exe, &config), "install" => svc::install(&exe, &config, &account), "uninstall" => svc::uninstall(&exe, &config), - _ => Err("unknown service action; use status/start/stop/enable/disable/install/uninstall".into()), + "prepare-account" => svc::prepare_account(&exe, &config), + _ => Err("unknown service action; use status/start/stop/enable/disable/install/uninstall/prepare-account".into()), }; } if service { return pdfcraft_hot_folder_service::run(&config, |path, stop| worker::run(path, || stop.load(std::sync::atomic::Ordering::Relaxed))); } if once { - worker::check_identity()?; - let state = hf::poll(&config, None)?; + let state = worker::once(&config)?; println!("{}", serde_json::to_string_pretty(&state).map_err(|e| e.to_string())?); return Ok(()); } diff --git a/apps/pdfcraft-hot-folder/src/worker.rs b/apps/pdfcraft-hot-folder/src/worker.rs index 722559055c..0d4f95e0f7 100644 --- a/apps/pdfcraft-hot-folder/src/worker.rs +++ b/apps/pdfcraft-hot-folder/src/worker.rs @@ -1,6 +1,10 @@ //! Background processing loop used by the manual worker and the system-service host. use pdfcraft_engine::hot_folder as hf; -use std::path::Path; +use std::io::Write; +use std::path::{Component, Path, PathBuf}; +use std::time::{Duration, Instant}; + +use crate::worker_process::{self, Limits, Outcome}; pub fn check_identity() -> Result<(), String> { #[cfg(unix)] @@ -11,7 +15,35 @@ pub fn check_identity() -> Result<(), String> { } pub fn run(config: &Path, should_stop: impl Fn() -> bool) -> Result<(), String> { + let Some(_lock) = prepare(config, false)? else { return Ok(()) }; + let limits = Limits::load(config).inspect_err(|error| report(config, Some(error)))?; + while !should_stop() { + if let Err(error) = pass(config, &limits, &should_stop) { + report(config, Some(&error)); + // A process that could not be reaped must not be left alive while another pass starts. + return Err(error); + } + let sleeping = Instant::now(); + while !should_stop() && sleeping.elapsed() < Duration::from_secs(1) { + std::thread::sleep(Duration::from_millis(100)); + } + } + Ok(()) +} + +pub fn once(config: &Path) -> Result { + let _lock = prepare(config, true)?.ok_or("another Hot Folder supervisor is already running")?; + hf::validate_structure(&hf::load_config(config)?, None)?; + pass(config, &Limits::load(config)?, &|| false)?; + hf::read_state(config) +} + +fn prepare(config: &Path, exclusive: bool) -> Result, String> { check_identity()?; + local_file(config)?; + for extension in ["worker.lock", "lock", "state.json", "state.pending.json"] { + local_file(&config.with_extension(extension))?; + } if !config.exists() { hf::initialize_config(config)?; } @@ -22,24 +54,138 @@ pub fn run(config: &Path, should_stop: impl Fn() -> bool) -> Result<(), String> .truncate(false) .open(config.with_extension("worker.lock")) .map_err(|e| e.to_string())?; - if lock.try_lock().is_err() { - return Ok(()); + match lock.try_lock() { + Ok(()) => Ok(Some(lock)), + Err(std::fs::TryLockError::WouldBlock) if !exclusive => Ok(None), + Err(error) => Err(format!("Cannot own the Hot Folder supervisor lock: {error}")), } - while !should_stop() { - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hf::poll_cancelable(config, None, &should_stop))) - .unwrap_or_else(|_| Err("processing failed safely; source files were retained".into())); - match result { - Ok(_) => { - hf::record_worker_error(config, None)?; - } - Err(error) => { - hf::record_worker_error(config, Some(&error))?; +} + +/// This is intentionally lexical until every ancestor has been checked: canonicalizing +/// a junction to an unavailable SMB share would already block the supervisor. +pub fn local_file(path: &Path) -> Result<(), String> { + if !path.is_absolute() || path.components().any(|component| matches!(component, Component::ParentDir | Component::CurDir)) { + return Err("worker configuration and journals need an absolute local path without '.' or '..'".into()); + } + #[cfg(windows)] + { + let drive = match path.components().next() { + Some(Component::Prefix(prefix)) => match prefix.kind() { + std::path::Prefix::Disk(drive) | std::path::Prefix::VerbatimDisk(drive) => drive, + _ => return Err("worker configuration and journals must be on a local disk, never a UNC share".into()), + }, + _ => return Err("worker configuration needs a local drive letter".into()), + }; + if winsafe::GetDriveType(Some(&format!("{}:\\", char::from(drive)))) != winsafe::co::DRIVE::FIXED { + return Err("worker configuration and journals must be on a fixed local disk, never a mapped network drive".into()); + } + } + let mut ancestor = PathBuf::new(); + for component in path.components() { + ancestor.push(component.as_os_str()); + if matches!(component, Component::Prefix(_)) { + continue; + } + match std::fs::symlink_metadata(&ancestor) { + Ok(metadata) => { + #[cfg(windows)] + let redirected = { + use std::os::windows::fs::MetadataExt; + metadata.file_attributes() & 0x400 != 0 // FILE_ATTRIBUTE_REPARSE_POINT, including junctions. + }; + #[cfg(not(windows))] + let redirected = false; + // macOS /var and /tmp are normal system symlinks. Only Windows guarantees + // local volumes here; Unix deployments must keep the config on local storage. + if metadata.file_type().is_symlink() && (cfg!(windows) || ancestor == path) || redirected { + return Err("worker configuration and journal paths cannot traverse symbolic links or reparse points".into()); + } + if ancestor == path && !metadata.is_file() { + return Err("worker configuration and journal targets must be regular files".into()); + } } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, + Err(error) => return Err(format!("Cannot inspect local worker path: {error}")), } - if !should_stop() { - std::thread::sleep(std::time::Duration::from_secs(1)); + } + Ok(()) +} + +fn report(config: &Path, error: Option<&str>) { + if let Err(write_error) = local_file(&config.with_extension("error.json")).and_then(|()| hf::record_worker_error(config, error)) { + // A full/unwritable error sidecar must never stop scheduling otherwise valid tasks. + let _ = writeln!(std::io::stderr(), "Cannot write Hot Folder diagnostics: {write_error}; {}", error.unwrap_or("worker pass completed")); + } +} + +fn pass(config: &Path, limits: &Limits, should_stop: &dyn Fn() -> bool) -> Result<(), String> { + pass_with(config, should_stop, |task, cancel| worker_process::run(config, &task.id, limits, cancel)) +} + +fn pass_with( + path: &Path, + should_stop: &dyn Fn() -> bool, + mut run_task: impl FnMut(&hf::Task, &dyn Fn() -> bool) -> Result, +) -> Result<(), String> { + let config = match local_file(path).and_then(|()| hf::load_config(path)).and_then(|config| hf::validate_structure(&config, None).map(|()| config)) + { + Ok(config) => config, + Err(error) => { + report(path, Some(&error)); + return Ok(()); // An atomic later edit can fix configuration without restarting the service. + } + }; + let state = match local_file(&path.with_extension("state.json")) + .and_then(|()| local_file(&path.with_extension("state.pending.json"))) + .and_then(|()| hf::read_state(path)) + { + Ok(state) => state, + Err(error) => { + report(path, Some(&error)); + return Ok(()); // Never replace a corrupt receipt journal with empty state. + } + }; + let mut failures = Vec::new(); + for task in &config.tasks { + if should_stop() { + break; + } + if !task.enabled + || task.start_at > hf::now() + || state.tasks.get(&task.id).is_some_and(|status| status.finished || status.blocked || status.next_check > hf::now()) + { + continue; + } + let snapshot = serde_json::to_vec(task).map_err(|e| e.to_string())?; + let checked = std::cell::Cell::new(None::); + let cancelled = std::cell::Cell::new(false); + let cancel = || { + if should_stop() || cancelled.get() { + return true; + } + if checked.get().is_none_or(|last| last.elapsed() >= Duration::from_millis(500)) { + checked.set(Some(Instant::now())); + let unchanged = local_file(path).and_then(|()| hf::load_config(path)).is_ok_and(|config| { + config + .tasks + .iter() + .any(|current| current.id == task.id && current.enabled && serde_json::to_vec(current).is_ok_and(|bytes| bytes == snapshot)) + }); + cancelled.set(!unchanged); + } + cancelled.get() + }; + match run_task(task, &cancel)? { + Outcome::Completed | Outcome::Cancelled => {} + Outcome::Failed(error) => { + failures.push(format!("{}: {error}", task.id)); + if let Err(error) = hf::record_task_failure(path, None, &task.id, &error) { + failures.push(format!("{}: cannot persist task failure: {error}", task.id)); + } + } } } + report(path, (!failures.is_empty()).then(|| failures.join("\n")).as_deref()); Ok(()) } @@ -58,4 +204,64 @@ mod tests { super::run(&path, || true).unwrap(); assert!(pdfcraft_engine::hot_folder::load_config(&path).unwrap().tasks.is_empty()); } + + #[test] + fn diagnostics_write_failure_and_failed_task_do_not_block_another_task() { + use super::*; + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("tasks.json"); + let mut tasks = Vec::new(); + for id in ["first", "second"] { + let input = dir.path().join(format!("{id}-in")); + let output = dir.path().join(format!("{id}-out")); + std::fs::create_dir(&input).unwrap(); + std::fs::create_dir(&output).unwrap(); + tasks.push(hf::Task { id: id.into(), name: id.into(), input, output, enabled: true, analysis_only: true, ..Default::default() }); + } + hf::save_config(&config, &hf::Config { tasks }, None).unwrap(); + std::fs::create_dir(config.with_extension("error.json")).unwrap(); + let mut called = Vec::new(); + pass_with(&config, &|| false, |task, _| { + called.push(task.id.clone()); + Ok(if task.id == "first" { Outcome::Failed("simulated timeout".into()) } else { Outcome::Completed }) + }) + .unwrap(); + assert_eq!(called, ["first", "second"]); + assert!(hf::read_state(&config).unwrap().tasks["first"].next_check > hf::now()); + } + + #[test] + fn changing_a_task_cancels_its_isolated_snapshot() { + use super::*; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("tasks.json"); + let input = dir.path().join("input"); + let output = dir.path().join("output"); + std::fs::create_dir(&input).unwrap(); + std::fs::create_dir(&output).unwrap(); + let mut config = hf::Config { + tasks: vec![hf::Task { id: "task".into(), name: "task".into(), input, output, enabled: true, analysis_only: true, ..Default::default() }], + }; + hf::save_config(&path, &config, None).unwrap(); + pass_with(&path, &|| false, |_, stop| { + config.tasks[0].enabled = false; + hf::save_config(&path, &config, None).unwrap(); + assert!(stop()); + Ok(Outcome::Cancelled) + }) + .unwrap(); + } + + #[test] + fn supervisor_paths_refuse_remote_paths_and_non_files() { + use std::path::Path; + let dir = tempfile::tempdir().unwrap(); + assert!(super::local_file(dir.path()).is_err()); + assert!(super::local_file(&dir.path().join("tasks.json")).is_ok()); + assert!(super::local_file(Path::new("relative.json")).is_err()); + #[cfg(windows)] + for path in [r"\\server\share\tasks.json", r"\\?\UNC\server\share\tasks.json", r"\\.\pipe\tasks.json"] { + assert!(super::local_file(Path::new(path)).is_err()); + } + } } diff --git a/apps/pdfcraft-hot-folder/src/worker_process.rs b/apps/pdfcraft-hot-folder/src/worker_process.rs new file mode 100644 index 0000000000..21852435bd --- /dev/null +++ b/apps/pdfcraft-hot-folder/src/worker_process.rs @@ -0,0 +1,476 @@ +//! One task pass per process. Only the child accesses document/network paths. +//! The supervisor never waits indefinitely for OCR, a network filesystem or a pipe. +use std::io::{Read, Write}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::time::{Duration, Instant}; + +use pdfcraft_engine::hot_folder as hf; +use serde::{Deserialize, Serialize}; + +const START: &[u8] = b"PDFCRAFT-TASK-1\n"; +const TICK: Duration = Duration::from_millis(100); +const STOP_GRACE: Duration = Duration::from_secs(3); +const REAP_LIMIT: Duration = Duration::from_secs(5); +const MAX_REPLY: u64 = 2048; + +#[derive(Clone, Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Limits { + pub task_timeout_seconds: u64, + pub memory_limit_mib: u64, +} + +impl Default for Limits { + fn default() -> Self { + Self { task_timeout_seconds: 300, memory_limit_mib: 4096 } + } +} + +impl Limits { + pub fn load(config: &Path) -> Result { + let path = config.with_extension("worker.json"); + crate::worker::local_file(&path)?; + let file = match std::fs::File::open(&path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Self::default()), + Err(error) => return Err(format!("Cannot read worker limits: {error}")), + }; + if file.metadata().map_err(|e| e.to_string())?.len() > 4096 { + return Err("worker limits exceed 4096 bytes".into()); + } + let limits: Self = serde_json::from_reader(file.take(4097)).map_err(|e| format!("Invalid worker limits: {e}"))?; + if !(10..=86400).contains(&limits.task_timeout_seconds) || !(256..=65536).contains(&limits.memory_limit_mib) { + return Err("worker limits require task_timeout_seconds 10..86400 and memory_limit_mib 256..65536".into()); + } + Ok(limits) + } + + fn policy(&self) -> Policy { + Policy { + timeout: Duration::from_secs(self.task_timeout_seconds), + grace: STOP_GRACE, + reap: REAP_LIMIT, + memory_bytes: self.memory_limit_mib.saturating_mul(1024 * 1024), + } + } +} + +#[derive(Clone, Copy)] +struct Policy { + timeout: Duration, + grace: Duration, + reap: Duration, + memory_bytes: u64, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Outcome { + Completed, + Cancelled, + Failed(String), +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Reply { + error: Option, +} + +/// A child cannot start work until the parent attached its Windows job and sent START. +/// No credential or privileged operation is exposed by this internal mode: it retains +/// the invoking account, requires a held supervisor lock, and validates the saved task. +pub fn child_main(config: &Path, task_id: &str) -> Result<(), String> { + crate::worker::check_identity()?; + let stop = read_start()?; + crate::worker::local_file(config)?; + let lock = std::fs::OpenOptions::new().read(true).write(true).open(config.with_extension("worker.lock")).map_err(|e| e.to_string())?; + match lock.try_lock() { + Err(std::fs::TryLockError::WouldBlock) => {} + _ => return Err("task children require an active supervisor".into()), + } + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hf::poll_task_cancelable(config, None, task_id, &|| stop.load(Ordering::Relaxed)))) + .unwrap_or_else(|_| Err("the isolated task failed safely; durable receipts and originals were retained".into())); + // Keep the reply below the anonymous pipe capacity. Diagnostics are JSON, never commands. + reply(result.err()) +} + +fn read_start() -> Result, String> { + let mut prefix = [0; START.len()]; + std::io::stdin().read_exact(&mut prefix).map_err(|_| "task child requires its supervisor input pipe")?; + if prefix != START { + return Err("invalid task child protocol".into()); + } + let stop = Arc::new(AtomicBool::new(false)); + let signal = stop.clone(); + std::thread::Builder::new() + .name("task-cancellation".into()) + .spawn(move || { + // A cancellation byte, closed pipe (including parent death), or read error stops work. + let _ = std::io::stdin().read(&mut [0]); + signal.store(true, Ordering::SeqCst); + }) + .map_err(|e| format!("Cannot start cancellation monitor: {e}"))?; + Ok(stop) +} + +fn reply(error: Option) -> Result<(), String> { + // 300 scalar values need at most 1800 JSON bytes, even for escaped control characters. + let reply = Reply { error: error.map(|message| message.chars().take(300).collect()) }; + let bytes = serde_json::to_vec(&reply).map_err(|e| e.to_string())?; + std::io::stdout().write_all(&bytes).and_then(|_| std::io::stdout().flush()).map_err(|e| e.to_string()) +} + +pub fn run(config: &Path, task_id: &str, limits: &Limits, should_stop: &dyn Fn() -> bool) -> Result { + let mut command = Command::new(std::env::current_exe().map_err(|e| e.to_string())?); + command.arg("--task-child").arg("--config").arg(config).arg("--task-id").arg(task_id); + supervise(&mut command, limits.policy(), should_stop) +} + +struct Running { + child: Child, + reply: std::sync::mpsc::Receiver, String>>, + reaped: bool, + termination_attempted: bool, + #[cfg(windows)] + job: Option, +} + +impl Running { + fn start(command: &mut Command, memory_bytes: u64) -> Result { + #[cfg(not(windows))] + let _ = memory_bytes; // Unix uses sampled resident memory, not a kernel commit limit. + command.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x08000000); // CREATE_NO_WINDOW. + } + let mut child = command.spawn().map_err(|e| format!("Cannot start isolated task: {e}"))?; + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let stdout = child.stdout.take(); + let mut running = Self { + child, + reply: receiver, + reaped: false, + termination_attempted: false, + #[cfg(windows)] + job: None, + }; + let stdout = stdout.ok_or("task reply pipe is unavailable")?; + std::thread::Builder::new() + .name("task-reply".into()) + .spawn(move || { + let mut bytes = Vec::new(); + let result = stdout.take(MAX_REPLY + 1).read_to_end(&mut bytes).map(|_| bytes).map_err(|e| e.to_string()); + let _ = sender.send(result); + }) + .map_err(|e| format!("Cannot start task reply monitor: {e}"))?; + #[cfg(windows)] + { + // JOB_OBJECT_LIMIT_JOB_MEMORY caps private committed memory before START, + // including rapid allocations between monitor ticks and any descendants. + // The safe wrapper also sets KILL_ON_JOB_CLOSE for supervisor crashes. + let options = processkit::ProcessGroupOptions::default().max_memory(memory_bytes); + let job = processkit::ProcessGroup::with_options(options).map_err(|e| format!("Cannot limit task process job: {e}"))?; + job.adopt_external(running.child.id()).map_err(|e| format!("Cannot isolate task process: {e}"))?; + running.job = Some(job); + } + running + .child + .stdin + .as_mut() + .ok_or("task control pipe is unavailable")? + .write_all(START) + .map_err(|e| format!("Cannot start task protocol: {e}"))?; + Ok(running) + } + + fn status(&mut self) -> Result, String> { + let status = self.child.try_wait().map_err(|e| format!("Cannot inspect isolated task: {e}"))?; + self.reaped |= status.is_some(); + Ok(status) + } + + fn cancel(&mut self) { + // Closing stdin is non-blocking and cannot fill a child-owned pipe. + self.child.stdin.take(); + } + + fn terminate(&mut self, reap_limit: Duration) -> Result<(), String> { + self.termination_attempted = true; + self.cancel(); + if self.status().is_ok_and(|status| status.is_some()) { + return Ok(()); + } + #[cfg(windows)] + self.job.take(); // Kill every member and release the job even if the child crashed. + let kill = self.child.kill(); + let started = Instant::now(); + loop { + if self.status()?.is_some() { + return Ok(()); + } + if started.elapsed() >= reap_limit { + return Err(format!( + "the operating system did not reap task PID {} within {} seconds after termination ({kill:?}); supervisor must restart", + self.child.id(), + reap_limit.as_secs() + )); + } + std::thread::sleep(TICK); + } + } + + fn outcome(&mut self, status: ExitStatus) -> Outcome { + if !status.success() { + return Outcome::Failed(format!("isolated task exited unexpectedly ({status}); source cleanup resumes only from durable receipts")); + } + #[cfg(windows)] + self.job.take(); // Also close inherited output handles in any unexpected descendants. + let bytes = match self.reply.recv_timeout(Duration::from_secs(1)) { + Ok(Ok(bytes)) if bytes.len() as u64 <= MAX_REPLY => bytes, + _ => return Outcome::Failed("isolated task returned an invalid, missing or oversized reply".into()), + }; + match serde_json::from_slice::(&bytes) { + Ok(Reply { error: None }) => Outcome::Completed, + Ok(Reply { error: Some(error) }) => Outcome::Failed(error), + Err(_) => Outcome::Failed("isolated task did not return a valid completion reply".into()), + } + } +} + +impl Drop for Running { + fn drop(&mut self) { + if !self.reaped && !self.termination_attempted { + // Covers startup failures and unwinding without leaving an unsupervised worker. + let _ = self.terminate(REAP_LIMIT); + } + } +} + +struct Memory { + #[cfg(not(windows))] + system: sysinfo::System, +} + +impl Memory { + fn new() -> Self { + Self { + #[cfg(not(windows))] + system: sysinfo::System::new(), + } + } + + fn bytes(&mut self, running: &Running) -> Result { + #[cfg(windows)] + { + // Peak committed bytes include paged-out allocations and descendants. + running + .job + .as_ref() + .ok_or("task process job is unavailable")? + .stats() + .map_err(|e| e.to_string())? + .peak_memory_bytes + .ok_or_else(|| "job memory information is unavailable".into()) + } + #[cfg(not(windows))] + { + let pid = sysinfo::Pid::from_u32(running.child.id()); + self.system.refresh_processes_specifics( + sysinfo::ProcessesToUpdate::Some(&[pid]), + true, + sysinfo::ProcessRefreshKind::nothing().with_memory(), + ); + self.system.process(pid).map(|process| process.memory()).ok_or_else(|| "process memory information is unavailable".into()) + } + } +} + +fn supervise(command: &mut Command, policy: Policy, should_stop: &dyn Fn() -> bool) -> Result { + if should_stop() { + return Ok(Outcome::Cancelled); + } + let mut running = Running::start(command, policy.memory_bytes)?; + let mut memory = Memory::new(); + let started = Instant::now(); + let mut cancellation: Option<(Instant, Outcome)> = None; + loop { + if let Some(status) = running.status()? { + return Ok(cancellation.map_or_else(|| running.outcome(status), |(_, outcome)| outcome)); + } + if cancellation.is_none() { + let reason = if should_stop() { + Some(Outcome::Cancelled) + } else if started.elapsed() >= policy.timeout { + Some(Outcome::Failed(format!( + "task pass exceeded {} seconds; increase task_timeout_seconds in the local worker limits for larger jobs", + policy.timeout.as_secs() + ))) + } else { + match memory.bytes(&running) { + Ok(bytes) if bytes > policy.memory_bytes => { + // A runaway allocation must not continue during the ordinary stop grace. + running.terminate(policy.reap)?; + return Ok(Outcome::Failed(format!( + "task exceeded the monitored memory limit ({} MiB)", + policy.memory_bytes / (1024 * 1024) + ))); + } + Ok(_) => None, + Err(error) => { + if let Some(status) = running.status()? { + return Ok(running.outcome(status)); + } + running.terminate(policy.reap)?; + return Ok(Outcome::Failed(format!("cannot safely monitor task memory: {error}"))); + } + } + }; + if let Some(reason) = reason { + running.cancel(); + cancellation = Some((Instant::now(), reason)); + } + } + if cancellation.as_ref().is_some_and(|(when, _)| when.elapsed() >= policy.grace) { + running.terminate(policy.reap)?; + return Ok(cancellation.map_or(Outcome::Cancelled, |(_, outcome)| outcome)); + } + std::thread::sleep(TICK); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn synthetic(scenario: &str) -> Command { + let mut command = Command::new(std::env::current_exe().unwrap()); + command.args(["--exact", "worker_process::tests::synthetic_child_entry", "--nocapture"]); + command.env("PDFCRAFT_WORKER_TEST_SCENARIO", scenario); + command + } + + fn policy() -> Policy { + Policy { + timeout: Duration::from_millis(700), + grace: Duration::from_millis(150), + reap: Duration::from_secs(3), + memory_bytes: 4 * 1024 * 1024 * 1024, + } + } + + #[test] + fn synthetic_child_entry() { + let Ok(scenario) = std::env::var("PDFCRAFT_WORKER_TEST_SCENARIO") else { return }; + let stop = read_start().unwrap(); + match scenario.as_str() { + "hang" => loop { + std::thread::park(); + }, + "crash" => std::process::exit(29), + "memory-refused" => { + let mut bytes = Vec::::new(); + std::process::exit(if bytes.try_reserve_exact(128 * 1024 * 1024).is_err() { 91 } else { 92 }); + } + "memory" => { + let bytes = vec![42_u8; 128 * 1024 * 1024]; + std::hint::black_box(&bytes); + loop { + std::thread::park(); + } + } + "stop" => { + while !stop.load(Ordering::Relaxed) { + std::thread::sleep(Duration::from_millis(10)); + } + } + _ => {} + } + // The harness writes banners to stdout, so a dedicated exit avoids its completion output. + // The supervisor tests use failure/stop paths and inspect the process lifetime, not this reply. + std::process::exit(0); + } + + #[test] + fn hung_task_is_killed_reaped_and_does_not_prevent_the_next_task() { + let started = Instant::now(); + let result = supervise(&mut synthetic("hang"), policy(), &|| false).unwrap(); + assert!(matches!(result, Outcome::Failed(message) if message.contains("exceeded"))); + assert!(started.elapsed() < Duration::from_secs(6)); + let next = supervise(&mut synthetic("crash"), policy(), &|| false).unwrap(); + assert!(matches!(next, Outcome::Failed(message) if message.contains("unexpectedly"))); + } + + #[test] + fn stop_cancels_a_cooperative_child_and_kills_a_hung_one_within_the_bound() { + for scenario in ["stop", "hang"] { + let started = Instant::now(); + let result = supervise(&mut synthetic(scenario), policy(), &|| started.elapsed() >= Duration::from_millis(200)).unwrap(); + assert_eq!(result, Outcome::Cancelled); + assert!(started.elapsed() < Duration::from_secs(5)); + } + } + + #[cfg(not(windows))] + #[test] + fn excessive_child_memory_is_detected_and_reaped() { + let mut policy = policy(); + policy.timeout = Duration::from_secs(10); + policy.memory_bytes = 64 * 1024 * 1024; + let result = supervise(&mut synthetic("memory"), policy, &|| false).unwrap(); + assert!(matches!(result, Outcome::Failed(ref message) if message.contains("memory limit")), "{result:?}"); + } + + #[cfg(windows)] + #[test] + fn windows_job_refuses_excess_commit_before_the_monitor_can_sample() { + let mut running = Running::start(&mut synthetic("memory-refused"), 64 * 1024 * 1024).unwrap(); + let started = Instant::now(); + let status = loop { + if let Some(status) = running.status().unwrap() { + break status; + } + assert!(started.elapsed() < Duration::from_secs(5), "allocation helper failed to finish"); + std::thread::sleep(TICK); + }; + // The helper uses fallible allocation and reports the actual refusal; + // an allocator abort or generic crash would not satisfy this assertion. + assert_eq!(status.code(), Some(91)); + } + + #[test] + fn stopped_supervisor_never_launches_a_child() { + assert_eq!(supervise(&mut Command::new("does-not-exist"), policy(), &|| true).unwrap(), Outcome::Cancelled); + } + + #[test] + fn limits_are_local_bounded_and_reject_unknown_or_out_of_range_values() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let path = config.with_extension("worker.json"); + assert_eq!(Limits::load(&config).unwrap().task_timeout_seconds, 300); + for invalid in [ + r#"{"task_timeout_seconds":0}"#, + r#"{"task_timeout_seconds":86401}"#, + r#"{"memory_limit_mib":255}"#, + r#"{"memory_limit_mib":65537}"#, + r#"{"misspelled_limit":123}"#, + ] { + std::fs::write(&path, invalid).unwrap(); + assert!(Limits::load(&config).is_err(), "{invalid}"); + } + std::fs::write(&path, vec![b' '; 4097]).unwrap(); + assert!(Limits::load(&config).unwrap_err().contains("4096")); + std::fs::write(&path, r#"{"task_timeout_seconds":900,"memory_limit_mib":2048}"#).unwrap(); + let limits = Limits::load(&config).unwrap(); + assert_eq!(limits.task_timeout_seconds, 900); + assert_eq!(limits.policy().memory_bytes, 2048 * 1024 * 1024); + } +} diff --git a/apps/pdfcraft-hot-folder/tests/worker.rs b/apps/pdfcraft-hot-folder/tests/worker.rs new file mode 100644 index 0000000000..982e3c79b9 --- /dev/null +++ b/apps/pdfcraft-hot-folder/tests/worker.rs @@ -0,0 +1,95 @@ +//! Exercise the shipped parent/child protocol, without installing a service or launching a GUI. +use std::path::Path; +use std::process::{Command, ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use pdfcraft_engine::{Session, hot_folder as hf, ocr}; + +fn run(args: &[&str], config: &Path) -> ExitStatus { + let mut command = Command::new(env!("CARGO_BIN_EXE_pdfcraft-hot-folder")); + command.args(args).arg("--config").arg(config).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + command.creation_flags(0x08000000); + } + let mut child = command.spawn().unwrap(); + let started = Instant::now(); + loop { + if let Some(status) = child.try_wait().unwrap() { + return status; + } + if started.elapsed() > Duration::from_secs(30) { + let _ = child.kill(); + // Poll, rather than an unbounded wait even in the failing test path. + let killing = Instant::now(); + while child.try_wait().unwrap().is_none() && killing.elapsed() < Duration::from_secs(5) { + std::thread::sleep(Duration::from_millis(50)); + } + panic!("supervised one-pass processing did not finish within 30 seconds"); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +#[test] +fn real_task_children_continue_after_an_unavailable_source_and_reuse_receipts_after_restart() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + #[cfg(unix)] + if rustix::process::geteuid().is_root() { + // Root processing is intentionally unsupported, including in test containers. + assert!(!run(&["--once"], &config).success()); + assert!(!config.exists()); + return; + } + let input = base.path().join("local-in"); + let output = base.path().join("local-out"); + std::fs::create_dir(&input).unwrap(); + std::fs::create_dir(&output).unwrap(); + let local = hf::Task { + id: "local".into(), + name: "Local".into(), + input: input.clone(), + output: output.clone(), + enabled: true, + interval_seconds: 1, + settle_seconds: 1, + settings: ocr::OcrSettings { recognize: false, auto_rotate: false, deskew: false, ..Default::default() }, + ..Default::default() + }; + let unavailable = hf::Task { + id: "unavailable".into(), + name: "Unavailable".into(), + input: base.path().join("missing-input"), + output: base.path().join("missing-output"), + ..local.clone() + }; + let settings = hf::Config { tasks: vec![unavailable, local] }; + hf::validate_structure(&settings, None).unwrap(); + std::fs::write(&config, serde_json::to_vec(&settings).unwrap()).unwrap(); + let source = Session::new().create_blank(100.0, 100.0, 1).unwrap(); + std::fs::write(input.join("page.pdf"), source.as_slice()).unwrap(); + assert!(run(&["--once"], &config).success()); + std::thread::sleep(Duration::from_millis(1100)); + assert!(run(&["--once"], &config).success()); + let state = hf::read_state(&config).unwrap(); + assert_eq!(state.tasks["local"].completed, 1); + assert!(state.tasks["unavailable"].retry_count > 0); + assert!(!state.tasks["unavailable"].blocked); + assert!(output.join("page.pdf").is_file()); + assert_eq!(std::fs::read(input.join("page.pdf")).unwrap(), *source); + std::thread::sleep(Duration::from_millis(1100)); + assert!(run(&["--once"], &config).success()); + assert_eq!(hf::read_state(&config).unwrap().tasks["local"].completed, 1); + assert_eq!(std::fs::read_dir(&output).unwrap().count(), 1); +} + +#[test] +fn internal_child_refuses_to_process_without_its_supervisor_pipe() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + assert!(!run(&["--task-child", "--task-id", "task"], &config).success()); + assert!(!config.exists()); + assert!(!config.with_extension("state.json").exists()); +} diff --git a/crates/automation/README.md b/crates/automation/README.md index 219d462283..428e677aaa 100644 --- a/crates/automation/README.md +++ b/crates/automation/README.md @@ -52,6 +52,16 @@ or fill colour. Names, rectangles and rotations require the single-field form. R Implemented: `initialize` (protocol 2025-06-18, 2025-03-26, 2024-11-05), `ping`, `tools/list` (with `readOnlyHint`/`destructiveHint` annotations), `tools/call` (JSON results also returned as `structuredContent`; images as `image/png`), and `resources/list`, `resources/templates/list` and `resources/read`. The resources expose the open documents read-only: `pdfcraft://doc/{doc}/info` (JSON), `…/text`, `…/page/{page}/text` and `…/page/{page}/image{?dpi}` (PNG, 1–600 dpi). Tool failures come back as `isError: true` results, so the agent can read and recover from them. +## Security regression tests + +Run `cargo test -p pdfcraft-automation --test hostile_args -- --nocapture` for the +hostile-argument and root-confinement sweep. On Windows without symbolic-link +privileges (error 1314), the sweep reports that only its symlink cases are unavailable; +all traversal and absolute-path cases still run. It never treats an ordinary path +inside the root as a symlink escape. Windows CI sets `PDFCRAFT_REQUIRE_SYMLINK_TESTS=1` +to require a real planted link and fail if that fixture cannot be created. Set the +same environment variable locally when checking the full symlink coverage. + ## Adding a tool 1. Add the engine capability first, with its tests (the tool is a thin adapter). diff --git a/crates/automation/src/hot_folder_service.rs b/crates/automation/src/hot_folder_service.rs index 87e9eb5b27..f585d83189 100644 --- a/crates/automation/src/hot_folder_service.rs +++ b/crates/automation/src/hot_folder_service.rs @@ -22,7 +22,8 @@ impl Automation { "disable" => service::set_enabled(false, &executable, &config).map_err(failed)?, "install" => service::install(&executable, &config, args.str("account")?).map_err(failed)?, "uninstall" => service::uninstall(&executable, &config).map_err(failed)?, - _ => return Err(failed("action must be status, start, stop, enable, disable, install or uninstall")), + "prepare-account" => service::prepare_account(&executable, &config).map_err(failed)?, + _ => return Err(failed("action must be status, start, stop, enable, disable, install, uninstall or prepare-account")), } serde_json::to_value(service::status(&executable, &config).map_err(failed)?).map_err(failed) } @@ -43,7 +44,7 @@ mod tests { let root = std::env::temp_dir(); let mut automation = Automation::new(); automation.root = Some(root); - for action in ["start", "stop", "enable", "disable", "install", "uninstall"] { + for action in ["start", "stop", "enable", "disable", "install", "uninstall", "prepare-account"] { let value = json!({"action":action,"executable":"unneeded","config":"unneeded","account":"root"}); assert!(automation.hot_folder_service(&Args(&value)).unwrap_err().to_string().contains("confined")); } diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index b21234a1a7..680375ca09 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -963,8 +963,8 @@ pub fn tools() -> Vec { .with(schema(json!({"config":{"type":"string"}, "task":{"type":"string"}}), &["config", "task"])), t("hot_folder_run_now", "Run Hot Folder at the next poll", "Make an enabled task due at its next poll, overriding its calendar schedule once. Successful receipts remain. The task must already be enabled and its start_at must be due; no worker is started.") .with(schema(json!({"config":{"type":"string"}, "task":{"type":"string"}}), &["config", "task"])), - t("hot_folder_service", "Manage Hot Folder system service", "Inspect or explicitly manage the system boot service matching executable and config. Changes require platform administrator rights and are unavailable under --root. Unix install requires a non-root account; Windows installation is performed by the optional MSI component. Nothing is installed implicitly.") - .destructive().with(schema(json!({"executable":{"type":"string"},"config":{"type":"string"},"action":{"type":"string","enum":["status","start","stop","enable","disable","install","uninstall"]},"account":{"type":"string"}}), &["executable","config","action"])), + t("hot_folder_service", "Manage Hot Folder system service", "Inspect or explicitly manage the system boot service matching executable and config. Changes require platform administrator rights and are unavailable under --root. Unix install requires a non-root account; Windows installation is performed by the optional MSI component; prepare-account configures its service SID and delayed failure recovery without changing logon credentials. Nothing is installed implicitly.") + .destructive().with(schema(json!({"executable":{"type":"string"},"config":{"type":"string"},"action":{"type":"string","enum":["status","start","stop","enable","disable","install","uninstall","prepare-account"]},"account":{"type":"string"}}), &["executable","config","action"])), t("hot_folder_source", "Connect Hot Folder source", "Manage the configured task's protected credentials or explicit Outlook device sign-in. No passwords or access tokens are returned. save_ftp_password reads a named environment variable; flow is the public object returned by outlook_begin. Stop the task before changing credentials. Credentials require the installer-provisioned service store. Outlook needs your own public-client application registration. No work starts implicitly.") .destructive().with(schema(json!({"config":{"type":"string"},"task":{"type":"string"},"action":{"type":"string","enum":["credential_status","save_ftp_password","forget_credentials","outlook_begin","outlook_poll","outlook_folders"]},"password_env":{"type":"string"},"flow":{"type":"object"},"parent":{"type":"string"}}), &["config","task","action"])), t("hot_folder_project", "Create Hot Folder project", "Create an open PDFCraft manifest containing ordered PDF/image paths. This is not an ABBYY proprietary project. All files must be inside the manifest's parent folder. Existing manifests are never overwritten; all paths honour --root.") diff --git a/crates/automation/tests/hostile_args.rs b/crates/automation/tests/hostile_args.rs index 08b51b6a58..f37fadb3bc 100644 --- a/crates/automation/tests/hostile_args.rs +++ b/crates/automation/tests/hostile_args.rs @@ -82,7 +82,7 @@ fn zoo() -> Vec { /// Paths that must never resolve inside the root. Absolute ones point into the test's own temp /// `base` (the root's parent), never at real system files or network shares, so a confinement /// bug can only touch what the test made. -fn escapes(base: &std::path::Path) -> Vec { +fn escapes(base: &std::path::Path, has_escape_link: bool) -> Vec { let mut out: Vec = vec![ "../outside.pdf".into(), "../../outside.pdf".into(), @@ -92,8 +92,10 @@ fn escapes(base: &std::path::Path) -> Vec { base.join("outside.pdf").to_string_lossy().into_owned(), base.join("new-outside.pdf").to_string_lossy().into_owned(), base.join("sub").join("new-outside.pdf").to_string_lossy().into_owned(), - "link-out/outside.pdf".into(), // through a planted symbolic link, when the system allows one ]; + if has_escape_link { + out.push("link-out/outside.pdf".into()); + } // Backslashes separate only on Windows; elsewhere this is an ordinary file name in the root. if cfg!(windows) { out.push(r"..\..\outside.pdf".into()); @@ -101,6 +103,32 @@ fn escapes(base: &std::path::Path) -> Vec { out } +/// Windows developers need Developer Mode or SeCreateSymbolicLinkPrivilege. Hosted Windows CI +/// sets PDFCRAFT_REQUIRE_SYMLINK_TESTS=1 so missing privileges fail instead of reducing coverage. +fn plant_escape_link(base: &Path, root: &Path) -> bool { + let link = root.join("link-out"); + #[cfg(unix)] + let result = std::os::unix::fs::symlink(base, &link); + #[cfg(windows)] + let result = std::os::windows::fs::symlink_dir(base, &link); + #[cfg(not(any(unix, windows)))] + let result = Err(std::io::Error::new(std::io::ErrorKind::Unsupported, "symlink fixture unavailable")); + match result { + Ok(()) => { + assert_eq!(link.canonicalize().expect("resolve planted link"), base.canonicalize().expect("resolve outside directory")); + true + } + Err(error) => { + let required = std::env::var_os("PDFCRAFT_REQUIRE_SYMLINK_TESTS").is_some_and(|v| v == "1"); + // ERROR_PRIVILEGE_NOT_HELD is an environment limitation, not a confinement result. + let unavailable = cfg!(windows) && error.raw_os_error() == Some(1314); + assert!(unavailable && !required, "cannot create required escape-link fixture: {error}"); + eprintln!("symlink escape cases unavailable: {error}; all traversal/absolute-path cases still run (Windows CI requires symlinks)"); + false + } + } +} + /// The properties of `def` whose values are paths, by name and whether they are a list. fn path_props(def: &pdfcraft_automation::ToolDef) -> Vec<(String, bool)> { let Some(props) = def.input_schema["properties"].as_object() else { return Vec::new() }; @@ -223,11 +251,9 @@ fn no_tool_escapes_the_root() { .expect("outside file"); let before = listing(&base); - // A link inside the root pointing out of it, where the system allows one to be made. - #[cfg(unix)] - let _ = std::os::unix::fs::symlink(&base, root.join("link-out")); - #[cfg(windows)] - let _ = std::os::windows::fs::symlink_dir(&base, root.join("link-out")); + // Without the Windows symlink privilege, this would be an ordinary path inside root. + // Only add it to the escape cases if the link actually exists. CI requires the real link. + let has_escape_link = plant_escape_link(&base, &root); let mut a = Automation::new().with_root(&root).expect("root"); @@ -247,7 +273,7 @@ fn no_tool_escapes_the_root() { for def in tools() { for (prop, is_list) in path_props(&def) { - for escape in escapes(&base) { + for escape in escapes(&base, has_escape_link) { let mut obj = plausible_with(&def, Some(doc)); obj.insert(prop.clone(), if is_list { json!([escape]) } else { json!(escape) }); let args = Value::Object(obj); diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 6c32a23872..3c18431264 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -56,6 +56,8 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } [target.'cfg(target_os = "windows")'.dependencies] keyring = { version = "3.6.3", default-features = false, features = ["windows-native"] } windows-permissions = "0.2.4" +windows-service = "0.8.1" +winsafe = { version = "0.0.29", features = ["advapi"] } [target.'cfg(unix)'.dependencies] rustix = { version = "1", features = ["process"] } diff --git a/crates/engine/src/hot_folder.rs b/crates/engine/src/hot_folder.rs index e7425e5cbd..1c4ae9c002 100644 --- a/crates/engine/src/hot_folder.rs +++ b/crates/engine/src/hot_folder.rs @@ -18,9 +18,11 @@ use crate::hot_folder_sources as sources; use crate::{Session, ocr}; const MAX_INPUT: u64 = 512 * 1024 * 1024; -const MAX_ENTRIES: usize = 10_000; const MAX_JSON: u64 = 16 * 1024 * 1024; +#[path = "hot_folder_journal.rs"] +pub(crate) mod journal; + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum Originals { @@ -109,6 +111,10 @@ pub struct TaskStatus { pub last_input: Option, pub last_output: Option, pub error: Option, + /// Consecutive recoverable errors; the delay is capped, never a lifetime retry limit. + pub retry_count: u32, + /// Stable output name timestamp across a killed/retried publication. + pub attempt_started_at: Option, } #[derive(Clone, Debug, Default, Serialize, Deserialize)] @@ -119,6 +125,8 @@ pub struct State { observations: BTreeMap, receipts: BTreeMap, pub log: Vec, + #[serde(default)] + observation_epochs: BTreeMap, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -135,6 +143,8 @@ struct Observation { fingerprint: String, since: u64, attempted: bool, + #[serde(default)] + receipt_key: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -240,13 +250,13 @@ pub fn initialize_config(path: &Path) -> Result<(), String> { directory(path.parent().ok_or("configuration needs a parent")?)?; let path = confined(path, None)?; if path.exists() { - return validate(&load_config(&path)?, None); + return validate_structure(&load_config(&path)?, None); } let mut temp = tempfile::NamedTempFile::new_in(path.parent().ok_or("configuration needs a parent")?).map_err(|e| e.to_string())?; temp.write_all(b"{\"tasks\":[]}\n").and_then(|_| temp.as_file().sync_all()).map_err(|e| e.to_string())?; match temp.persist_noclobber(&path) { Ok(_) => sync_parent(&path), - Err(e) if e.error.kind() == std::io::ErrorKind::AlreadyExists => validate(&load_config(&path)?, None), + Err(e) if e.error.kind() == std::io::ErrorKind::AlreadyExists => validate_structure(&load_config(&path)?, None), Err(e) => Err(e.to_string()), } } @@ -263,6 +273,9 @@ pub fn state_path(config: &Path) -> PathBuf { pub fn read_state(config: &Path) -> Result { let path = state_path(config); + if let Some(state) = journal::pending(&path)? { + return Ok(state); + } if !path.exists() { return Ok(State::default()); } @@ -324,7 +337,11 @@ pub fn confined(path: &Path, root: Option<&Path>) -> Result { fn lexically_inside(path: &Path, root: &Path) -> bool { #[cfg(windows)] { - let normalize = |p: &Path| p.to_string_lossy().trim_start_matches(r"\\?\").replace('/', "\\").to_ascii_lowercase(); + let normalize = |p: &Path| { + let raw = p.to_string_lossy().replace('/', "\\"); + let path = if let Some(unc) = raw.strip_prefix(r"\\?\UNC\") { format!(r"\\{unc}") } else { raw.trim_start_matches(r"\\?\").to_owned() }; + path.to_ascii_lowercase() + }; let path = normalize(path); let root = normalize(root); path == root || path.starts_with(&format!("{}\\", root.trim_end_matches('\\'))) @@ -333,9 +350,86 @@ fn lexically_inside(path: &Path, root: &Path) -> bool { path.starts_with(root) } +/// Validate scheduler/configuration structure without probing source or output volumes. +/// The supervisor only uses this form; runtime checks happen in the selected task child. +pub fn validate_structure(config: &Config, root: Option<&Path>) -> Result<(), String> { + if config.tasks.len() > 32 { + return Err("at most 32 hot folder tasks are allowed".into()); + } + let mut ids = BTreeSet::new(); + let mut sources = Vec::new(); + let mut destinations = Vec::new(); + for t in &config.tasks { + if t.id.is_empty() || t.id.len() > 100 || !t.id.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') || !ids.insert(&t.id) { + return Err("task IDs must be unique, using 1..100 ASCII letters, digits, '-' or '_'".into()); + } + if t.name.trim().is_empty() || t.name.len() > 200 { + return Err("task name is required (at most 200 bytes)".into()); + } + if !t.pdf && !t.images { + return Err("choose PDF, images, or both".into()); + } + if !(1..=86400).contains(&t.interval_seconds) || !(1..=300).contains(&t.settle_seconds) { + return Err("check interval must be 1..86400 seconds; settling time 1..300 seconds".into()); + } + t.schedule.validate()?; + t.preprocessing.validate()?; + t.output_plan.validate_settings()?; + let settings = &t.settings; + if !settings.dpi.is_finite() + || !(72.0..=600.0).contains(&settings.dpi) + || !settings.crop_margin.is_finite() + || !(0.0..=20.0).contains(&settings.crop_margin) + || !ocr::LANGUAGES.iter().any(|language| language.0 == settings.language) + { + return Err("invalid OCR language, resolution or crop margin".into()); + } + let input = lexical_path(&t.input, root)?; + let output = lexical_path(&t.output, root)?; + let mut outputs = vec![output]; + if t.originals == Originals::Move { + outputs.push(lexical_path(&t.archive, root)?); + } + if let Some(folder) = &t.blank_archive { + outputs.push(lexical_path(folder, root)?); + } + for step in &t.output_plan.steps { + if let Some(folder) = &step.folder { + outputs.push(lexical_path(folder, root)?); + } + } + if outputs.iter().any(|out| lexically_inside(out, &input) || lexically_inside(&input, out)) { + return Err("input, output and archive folders must not contain one another".into()); + } + if t.enabled { + sources.push(input); + destinations.extend(outputs); + } + } + for (index, source) in sources.iter().enumerate() { + if sources.iter().skip(index + 1).chain(&destinations).any(|other| lexically_inside(source, other) || lexically_inside(other, source)) { + return Err("task input/output folders must not overlap".into()); + } + } + Ok(()) +} + +fn lexical_path(path: &Path, root: Option<&Path>) -> Result { + if path.as_os_str().is_empty() || path.as_os_str().len() > 32767 || path.components().any(|c| matches!(c, std::path::Component::ParentDir)) { + return Err("choose an absolute folder without parent traversal".into()); + } + let absolute = if path.is_absolute() { path.to_owned() } else { root.ok_or("folder paths must be absolute")?.join(path) }; + let absolute: PathBuf = absolute.components().filter(|c| !matches!(c, std::path::Component::CurDir)).collect(); + if root.is_some_and(|root| !lexically_inside(&absolute, root)) { + return Err("hot folder path is outside the allowed directory".into()); + } + Ok(absolute) +} + /// All sources and destinations must already exist and be disjoint, including across tasks. /// This prevents recursive output loops and destructive source policies affecting another task. pub fn validate(config: &Config, root: Option<&Path>) -> Result<(), String> { + validate_structure(config, root)?; if config.tasks.len() > 32 { return Err("at most 32 hot folder tasks are allowed".into()); } @@ -372,6 +466,9 @@ pub fn validate(config: &Config, root: Option<&Path>) -> Result<(), String> { { return Err("invalid OCR language, resolution or crop margin".into()); } + if !t.enabled { + continue; + } let input = confined(&t.input, root)?; if let sources::SourceConfig::Outlook(source) = &t.source && !matches!(source.action, sources::SourceAction::Keep) @@ -483,6 +580,12 @@ fn save_json(path: &Path, value: &T) -> Result<(), String> { std::os::unix::fs::chown(temp.path(), Some(metadata.uid()), Some(metadata.gid())).map_err(|e| e.to_string())?; temp.as_file().set_permissions(metadata.permissions()).map_err(|e| e.to_string())?; } + #[cfg(unix)] + if previous.is_none() && rustix::process::geteuid().as_raw() == 0 { + use std::os::unix::fs::MetadataExt; + let owner = fs::metadata(parent).map_err(|e| e.to_string())?; + std::os::unix::fs::fchown(temp.as_file(), Some(owner.uid()), Some(owner.gid())).map_err(|e| e.to_string())?; + } #[cfg(not(unix))] let _ = previous; temp.write_all(&bytes).and_then(|_| temp.as_file().sync_all()).map_err(|e| e.to_string())?; @@ -499,55 +602,137 @@ pub fn poll(config_path: &Path, root: Option<&Path>) -> Result { /// Service-aware form: cancellation between files and OCR pages discards partial outputs. pub fn poll_cancelable(config_path: &Path, root: Option<&Path>, should_stop: &dyn Fn() -> bool) -> Result { + poll_selected(config_path, root, None, should_stop) +} + +/// A supervisor can isolate one task in a killable child process. Only that task's +/// directories/remote sources are inspected; other tasks may point at offline shares. +pub fn poll_task_cancelable(config_path: &Path, root: Option<&Path>, task_id: &str, should_stop: &dyn Fn() -> bool) -> Result { + poll_selected(config_path, root, Some(task_id), should_stop) +} + +fn poll_selected(config_path: &Path, root: Option<&Path>, selected: Option<&str>, should_stop: &dyn Fn() -> bool) -> Result { let path = confined(config_path, root)?; - let state_file = confined(&state_path(&path), root)?; let lock_path = confined(&path.with_extension("lock"), root)?; let lock = OpenOptions::new().create(true).truncate(false).read(true).write(true).open(lock_path).map_err(|e| e.to_string())?; lock.try_lock().map_err(|_| "another Hot Folder worker is processing this configuration".to_string())?; let config = load_config(&path)?; - validate(&config, root)?; + validate_structure(&config, root)?; + if selected.is_some_and(|id| !config.tasks.iter().any(|t| t.id == id)) { + return Err("no such Hot Folder task".into()); + } let mut state = read_state(&path)?; let current = now(); for task in &config.tasks { if should_stop() { break; } - if !task.enabled || current < task.start_at { + if selected.is_some_and(|id| id != task.id) || !task.enabled || current < task.start_at { continue; } let status = state.tasks.entry(task.id.clone()).or_default(); - if !task.once && task.schedule != Schedule::Interval && status.next_check == 0 && !status.batch_active { + let initialized_schedule = !task.once && task.schedule != Schedule::Interval && status.next_check == 0 && !status.batch_active; + if initialized_schedule { status.next_check = task.schedule.next_after(current.max(task.start_at).saturating_sub(1))?; } if status.finished || status.blocked || current < status.next_check { + if initialized_schedule { + state.updated_at = current; + journal::save(&path, &mut state)?; + } continue; } status.next_check = current.saturating_add(task.interval_seconds); status.batch_active = true; - if let Err(error) = step(task, &mut state, current, &path, root, should_stop) { - let status = state.tasks.entry(task.id.clone()).or_default(); - status.error = Some(error.clone()); - let snapshot = serde_json::to_vec(task).map_err(|e| e.to_string())?; - let still_enabled = task_unchanged(&path, &task.id, &snapshot); - status.blocked = task.output_plan.needs_batch() && !should_stop() && still_enabled; - status.failed = status.failed.saturating_add(1); - state.log.push(LogEntry { at: now(), task: task.id.clone(), input: status.last_input.clone(), output: None, error: Some(error) }); - } - if state.log.len() > 1000 { - state.log.drain(..state.log.len().saturating_sub(1000)); - } - let status = state.tasks.entry(task.id.clone()).or_default(); - if status.waiting == 0 { - status.batch_active = false; - if !task.once && task.schedule != Schedule::Interval { - status.next_check = task.schedule.next_after(now())?; + status.attempt_started_at.get_or_insert(current); + let completed_before = status.completed; + // Persist the publication identity before any source/export I/O. A child killed + // after publishing but before its receipt will reuse date/time-based filenames. + journal::save(&path, &mut state)?; + let result = validate_runtime(task, &config, root).and_then(|()| step(task, &mut state, current, &path, root, should_stop)); + match result { + Err(error) => { + let snapshot = serde_json::to_vec(task).map_err(|e| e.to_string())?; + if !should_stop() && task_unchanged(&path, &task.id, &snapshot) { + task_failure(&mut state, &task.id, error, current); + } + } + Ok(()) => { + let status = state.tasks.entry(task.id.clone()).or_default(); + status.retry_count = 0; + status.error = None; + if status.completed > completed_before || status.waiting == 0 { + status.attempt_started_at = None; + } + if status.waiting == 0 { + status.batch_active = false; + if !task.once && task.schedule != Schedule::Interval { + status.next_check = task.schedule.next_after(now())?; + } + } } } state.updated_at = now(); - save_json(&state_file, &state)?; + journal::save(&path, &mut state)?; } - state.updated_at = current; - save_json(&state_file, &state)?; + Ok(state) +} + +fn validate_runtime(task: &Task, config: &Config, root: Option<&Path>) -> Result<(), String> { + validate(&Config { tasks: vec![task.clone()] }, root)?; + // Manifest contents may change after configuration was saved. Resolve only this + // task's originals; comparing with other task boundaries must never contact SMB. + let originals = sources::local_paths(&task.source, root)?; + for other in config.tasks.iter().filter(|other| other.enabled && other.id != task.id) { + let mut boundaries = vec![lexical_path(&other.input, root)?, lexical_path(&other.output, root)?]; + if other.originals == Originals::Move { + boundaries.push(lexical_path(&other.archive, root)?); + } + if let Some(folder) = &other.blank_archive { + boundaries.push(lexical_path(folder, root)?); + } + for step in &other.output_plan.steps { + if let Some(folder) = &step.folder { + boundaries.push(lexical_path(folder, root)?); + } + } + if originals.iter().any(|source| boundaries.iter().any(|boundary| lexically_inside(source, boundary) || lexically_inside(boundary, source))) { + return Err("project originals overlap another active task's folders; originals retained".into()); + } + } + Ok(()) +} + +fn task_failure(state: &mut State, id: &str, error: String, current: u64) { + let error: String = error.chars().take(2048).collect(); + let status = state.tasks.entry(id.to_owned()).or_default(); + status.error = Some(error.clone()); + status.failed = status.failed.saturating_add(1); + status.retry_count = status.retry_count.saturating_add(1); + let delay = 5_u64.saturating_mul(1_u64 << status.retry_count.saturating_sub(1).min(8)).min(900); + status.next_check = current.saturating_add(delay); + state.log.push(LogEntry { at: current, task: id.to_owned(), input: status.last_input.clone(), output: None, error: Some(error) }); +} + +/// Record a timed-out/crashed child only after it was killed and reaped. No task +/// filesystem is touched here, so an unavailable share cannot stall the supervisor. +pub fn record_task_failure(config: &Path, root: Option<&Path>, task_id: &str, message: &str) -> Result { + let path = confined(config, root)?; + if !load_config(&path)?.tasks.iter().any(|task| task.id == task_id) { + return Err("no such Hot Folder task".into()); + } + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(confined(&path.with_extension("lock"), root)?) + .map_err(|e| e.to_string())?; + lock.try_lock().map_err(|_| "task still owns its processing lock; reap it before recording failure")?; + let mut state = read_state(&path)?; + task_failure(&mut state, task_id, message.chars().take(2048).collect(), now()); + state.updated_at = now(); + journal::save(&path, &mut state)?; Ok(state) } @@ -566,41 +751,45 @@ fn fingerprint(path: &Path) -> Result { Ok(format!("{}:{modified}", meta.len())) } -fn files(task: &Task, root: Option<&Path>) -> Result, String> { +/// Walk with at most 33 directory handles and no list of completed input paths. +fn visit_files( + task: &Task, + root: Option<&Path>, + stop: &dyn Fn() -> bool, + visit: &mut dyn FnMut(PathBuf) -> Result<(), String>, +) -> Result<(), String> { let input = confined(&task.input, root)?; - let mut pending = vec![(input.clone(), 0usize)]; - let mut out = Vec::new(); - let mut count = 0usize; - while let Some((dir, depth)) = pending.pop() { - for entry in fs::read_dir(dir).map_err(|e| e.to_string())? { - count = count.saturating_add(1); - if count > MAX_ENTRIES { - return Err("input folder exceeds 10,000 entries".into()); - } - let entry = entry.map_err(|e| e.to_string())?; - let kind = entry.file_type().map_err(|e| e.to_string())?; - if kind.is_symlink() { - continue; - } - let path = entry.path(); - if !confined(&path, root)?.starts_with(&input) { - return Err("input escaped its folder".into()); + let mut stack = vec![fs::read_dir(&input).map_err(|e| e.to_string())?]; + while let Some(entries) = stack.last_mut() { + if stop() { + return Err("task stopped during folder enumeration".into()); + } + let Some(entry) = entries.next() else { + stack.pop(); + continue; + }; + let entry = entry.map_err(|e| e.to_string())?; + let kind = entry.file_type().map_err(|e| e.to_string())?; + if kind.is_symlink() { + continue; + } + let path = entry.path(); + if !confined(&path, root)?.starts_with(&input) { + return Err("input escaped its folder".into()); + } + if kind.is_dir() && task.recursive { + if stack.len() >= 33 { + return Err("input folder nesting exceeds 32 levels".into()); } - if kind.is_dir() && task.recursive { - if depth >= 32 { - return Err("input folder nesting exceeds 32 levels".into()); - } - pending.push((path, depth + 1)); - } else if kind.is_file() { - let ext = path.extension().unwrap_or_default().to_string_lossy().to_ascii_lowercase(); - if task.pdf && ext == "pdf" || task.images && matches!(ext.as_str(), "png" | "jpg" | "jpeg" | "tif" | "tiff" | "bmp") { - out.push(path); - } + stack.push(fs::read_dir(path).map_err(|e| e.to_string())?); + } else if kind.is_file() { + let ext = path.extension().unwrap_or_default().to_string_lossy().to_ascii_lowercase(); + if task.pdf && ext == "pdf" || task.images && matches!(ext.as_str(), "png" | "jpg" | "jpeg" | "tif" | "tiff" | "bmp") { + visit(path)?; } } } - out.sort(); - Ok(out) + Ok(()) } struct Prepared { @@ -641,9 +830,6 @@ fn remove_staged(path: &Path, expected: &str) -> Result<(), String> { } fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: Option<&Path>, should_stop: &dyn Fn() -> bool) -> Result<(), String> { - if state.observations.len() >= MAX_ENTRIES || state.receipts.len() >= MAX_ENTRIES { - return Err("task journal reached 10,000 entries; archive it before continuing".into()); - } let input = confined(&task.input, root)?; let output = confined(&task.output, root)?; let snapshot = serde_json::to_vec(task).map_err(|e| e.to_string())?; @@ -655,7 +841,7 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: let mut remote = BTreeMap::new(); let mut acknowledgement_proofs = Vec::new(); let candidates = if matches!(task.source, sources::SourceConfig::Local) { - files(task, root)? + Vec::new() } else { let staged = sources::stage(&task.source, &input, root, &task_stopped, &store)?; if staged.has_more && task.output_plan.needs_batch() { @@ -667,8 +853,9 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: for item in &staged.pending_acknowledgements { let path = confined(&input.join(&item.staged_relative), root)?; let key = format!("{}:{}:{}", task.id, path.display(), item.sha256); - let receipt = state.receipts.get(&key).ok_or("pending source acknowledgement has no saved-output receipt; original retained")?; - verify_outputs(receipt, root)?; + let receipt = + journal::receipt(config_path, state, &key)?.ok_or("pending source acknowledgement has no saved-output receipt; original retained")?; + verify_outputs(&receipt, root)?; } let mut items = staged.items; items.extend(staged.pending_acknowledgements); @@ -677,8 +864,8 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: for item in &items { let path = confined(&input.join(&item.staged_relative), root)?; let key = format!("{}:{}:{}", task.id, path.display(), item.sha256); - if let Some(receipt) = state.receipts.get(&key) { - verify_outputs(receipt, root)?; + if let Some(receipt) = journal::receipt(config_path, state, &key)? { + verify_outputs(&receipt, root)?; acknowledgement_proofs.push(receipt.clone()); } } @@ -693,7 +880,7 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: } let key = format!("{}:{}", task.id, path.display()); // Recover a failed/interrupted acknowledgement without rerunning OCR or saving duplicates. - if state.receipts.contains_key(&format!("{key}:{}", item.sha256)) { + if journal::receipt(config_path, state, &format!("{key}:{}", item.sha256))?.is_some() { for receipt in &acknowledgement_proofs { verify_outputs(receipt, root)?; } @@ -712,26 +899,58 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: let mut ready = Vec::new(); let mut waiting = 0usize; let batch = task.output_plan.needs_batch(); - for path in candidates { + let mut visited = 0_usize; + let mut consider = |path: PathBuf| -> Result<(), String> { let key = format!("{}:{}", task.id, path.display()); let stamp = fingerprint(&path).map_err(|e| format!("{}: {e}", path.display()))?; - let observation = - state.observations.entry(key.clone()).or_insert_with(|| Observation { fingerprint: stamp.clone(), since: current, attempted: false }); - if observation.fingerprint != stamp { - *observation = Observation { fingerprint: stamp.clone(), since: current, attempted: false }; - } - let pending_cleanup = state - .receipts - .iter() - .any(|(receipt_key, receipt)| receipt_key.starts_with(&format!("{key}:")) && receipt.source_stamp == stamp && receipt.cleanup.is_some()); + let previous = journal::observation(config_path, state, &key)?; + let changed = previous.as_ref().is_none_or(|observation| observation.fingerprint != stamp); + let observation = if changed { + Observation { fingerprint: stamp.clone(), since: current, attempted: false, receipt_key: None } + } else { + previous.ok_or("source observation unavailable")? + }; + let pending_cleanup = if let Some(receipt_key) = &observation.receipt_key { + journal::receipt(config_path, state, receipt_key)?.is_some_and(|r| r.source_stamp == stamp && r.cleanup.is_some()) + } else { + state.receipts.iter().any(|(receipt_key, receipt)| { + receipt_key.starts_with(&format!("{key}:")) && receipt.source_stamp == stamp && receipt.cleanup.is_some() + }) + }; if observation.attempted && !pending_cleanup { - continue; + return Ok(()); } waiting = waiting.saturating_add(1); - if (batch || ready.is_empty()) && current.saturating_sub(observation.since) >= task.settle_seconds { - ready.push((path, key, stamp)); + let selected = current.saturating_sub(observation.since) >= task.settle_seconds; + if changed || selected { + state.observations.insert(key.clone(), observation); + } + if selected { + if batch { + if ready.len() < 129 { + ready.push((path, key, stamp)); + } + } else if ready.first().is_none_or(|(first, _, _)| path < *first) { + ready.clear(); + ready.push((path, key, stamp)); + } + } + visited = visited.saturating_add(1); + if visited.is_multiple_of(256) { + journal::save(config_path, state)?; + } + Ok(()) + }; + if matches!(task.source, sources::SourceConfig::Local) { + visit_files(task, root, should_stop, &mut consider)?; + } else { + for path in candidates { + consider(path)?; } } + if matches!(task.source, sources::SourceConfig::Local) { + ready.sort_by(|a, b| a.0.cmp(&b.0)); + } let status = state.tasks.entry(task.id.clone()).or_default(); status.waiting = waiting; if ready.is_empty() && waiting == 0 && task.once { @@ -739,18 +958,29 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: } // A grouped result includes the complete currently visible queue; never consume an // unfinished copy or publish a misleading partial group while another input is settling. + if ready.len() > 128 { + status.blocked = true; + return Err("combined batch exceeds 128 files; split the task or use one output per file".into()); + } if ready.is_empty() || batch && ready.len() != waiting { return Ok(()); } - if ready.len() > 128 { - return Err("combined batch exceeds 128 files; split the task or use one output per file".into()); + // Model installation is an environment failure, not a permanent bad input. + if !task.analysis_only && (task.settings.recognize || task.settings.auto_rotate || task.settings.deskew || task.settings.auto_crop) { + ocr::engine()?; } + let publication_time = state.tasks.get(&task.id).and_then(|status| status.attempt_started_at).unwrap_or(current); let mut prepared = Vec::new(); + let mut prepared_bytes = 0_u64; let mut total = 0u64; for (source, key, stamp) in ready { if task_stopped() { return Err("task stopped; no partial output was saved".into()); } + if !state.observations.contains_key(&key) { + let observation = journal::observation(config_path, state, &key)?.ok_or("source observation unavailable")?; + state.observations.insert(key.clone(), observation); + } state.tasks.entry(task.id.clone()).or_default().last_input = Some(source.clone()); let file = File::open(&source).map_err(|e| e.to_string())?; let length = file.metadata().map_err(|e| e.to_string())?.len(); @@ -759,6 +989,7 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: if let Some(o) = state.observations.get_mut(&key) { o.attempted = true; } + state.tasks.entry(task.id.clone()).or_default().blocked = batch; return Err("input or combined batch must contain 1 byte to 512 MiB; sources retained".into()); } let mut bytes = Vec::new(); @@ -768,14 +999,15 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: } let hash: String = Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect(); let receipt_key = format!("{key}:{hash}"); - if let Some(receipt) = state.receipts.get(&receipt_key).cloned() { + if let Some(receipt) = journal::receipt(config_path, state, &receipt_key)? { + state.receipts.insert(receipt_key.clone(), receipt.clone()); if let Some(cleanup) = &receipt.cleanup { verify_outputs(&receipt, root)?; finish_original(&source, &hash, cleanup, task, root, &task_stopped)?; if let Some(receipt) = state.receipts.get_mut(&receipt_key) { receipt.cleanup = None; } - save_json(&state_path(config_path), state)?; + journal::save(config_path, state)?; } if let Some(o) = state.observations.get_mut(&key) { o.attempted = true; @@ -783,13 +1015,14 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: state.tasks.entry(task.id.clone()).or_default().waiting = state.tasks.get(&task.id).map_or(0, |s| s.waiting.saturating_sub(1)); continue; } - let processed = match prepare(task, &source, bytes.clone(), config_path, &task_stopped) { + let processed = match prepare(task, &source, bytes, config_path, MAX_INPUT.saturating_sub(prepared_bytes), publication_time, &task_stopped) { Ok(bytes) => bytes, Err(e) => { if !task_stopped() && let Some(o) = state.observations.get_mut(&key) { o.attempted = true; + state.tasks.entry(task.id.clone()).or_default().blocked = batch; } return Err(e); } @@ -797,6 +1030,16 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: if fingerprint(&source)? != stamp { return Err("input changed during processing; sources retained".into()); } + let expanded = + processed.document.as_ref().map_or(0, |v| v.len() as u64).saturating_add(processed.blank_pages.as_ref().map_or(0, |v| v.len() as u64)); + prepared_bytes = prepared_bytes.saturating_add(expanded); + if prepared_bytes > MAX_INPUT { + state.tasks.entry(task.id.clone()).or_default().blocked = batch; + if let Some(observation) = state.observations.get_mut(&key) { + observation.attempted = true; + } + return Err("prepared batch exceeds 512 MiB; lower resolution or split the batch; originals retained".into()); + } prepared.push(Prepared { relative: remote.get(&source).map(|item| item.relative.clone()).unwrap_or(source.strip_prefix(&input).map_err(|e| e.to_string())?.into()), remote: remote.get(&source).cloned(), @@ -836,7 +1079,7 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: step.language = task.settings.language.clone(); } } - output::save(&inputs, &output, &plan, &task.name, current, root, &stopped) + output::save(&inputs, &output, &plan, &task.name, publication_time, root, &stopped) }; let mut result = match primary { Ok(result) => result, @@ -846,9 +1089,10 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: } for file in &prepared { if let Some(o) = state.observations.get_mut(&file.key) { - o.attempted = !stopped(); + o.attempted = !stopped() && !error.retryable; } } + state.tasks.entry(task.id.clone()).or_default().blocked = batch && !error.retryable && !stopped(); return Err(format!("{}; all originals retained", error.message)); } }; @@ -859,7 +1103,7 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: .collect(); if !blanks.is_empty() { let plan = OutputPlan { steps: vec![output::SaveStep { name_template: "{name}_blank_pages".into(), ..Default::default() }] }; - match output::save(&blanks, archive, &plan, &task.name, current, root, &stopped) { + match output::save(&blanks, archive, &plan, &task.name, publication_time, root, &stopped) { Ok(saved) => { result.outputs.extend(saved.outputs); result.proofs.extend(saved.proofs); @@ -877,9 +1121,10 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: } for file in &prepared { if let Some(o) = state.observations.get_mut(&file.key) { - o.attempted = !stopped(); + o.attempted = !stopped() && !error.retryable; } } + state.tasks.entry(task.id.clone()).or_default().blocked = batch && !error.retryable && !stopped(); return Err(format!("blank-page archive failed: {}; all originals retained", error.message)); } } @@ -912,6 +1157,7 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: ); if let Some(o) = state.observations.get_mut(&file.key) { o.attempted = true; + o.receipt_key = Some(format!("{}:{}", file.key, file.hash)); } state.log.push(LogEntry { at: now(), @@ -933,15 +1179,17 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: status.error = None; status.waiting = status.waiting.saturating_sub(prepared.len()); // All outputs and their receipts are durable before any source move/delete is attempted. - save_json(&state_path(config_path), state)?; + journal::save(config_path, state)?; for file in prepared { if stopped() { return Err("task stopped after outputs were saved; remaining original actions will resume on retry".into()); } let receipt_key = format!("{}:{}", file.key, file.hash); - if file.remote.is_some() || state.receipts.get(&receipt_key).is_some_and(|r| r.cleanup.is_some()) { - verify_outputs(state.receipts.get(&receipt_key).ok_or("output receipt unavailable; original retained")?, root)?; + let receipt = journal::receipt(config_path, state, &receipt_key)?.ok_or("output receipt unavailable; original retained")?; + if file.remote.is_some() || receipt.cleanup.is_some() { + verify_outputs(&receipt, root)?; } + state.receipts.insert(receipt_key.clone(), receipt); if let Some(remote) = &file.remote { // A previous ready attachment's output may have changed during lengthy OCR // of this final attachment. Recheck the whole pending group at the action. @@ -958,7 +1206,7 @@ fn step(task: &Task, state: &mut State, current: u64, config_path: &Path, root: if let Some(receipt) = state.receipts.get_mut(&key) { receipt.cleanup = None; } - save_json(&state_path(config_path), state)?; + journal::save(config_path, state)?; } } Ok(()) @@ -969,26 +1217,42 @@ fn prepare( source: &Path, bytes: Vec, config_path: &Path, + budget: u64, + publication_time: u64, should_stop: &dyn Fn() -> bool, ) -> Result { + let timestamp = i64::try_from(publication_time).map_err(|_| "invalid publication timestamp")?; let name = source.file_name().unwrap_or_default().to_string_lossy(); let pdf = if source.extension().is_some_and(|e| e.eq_ignore_ascii_case("pdf")) { Arc::new(bytes) } else { - Session::new().create_from_images(&[(name.to_string(), bytes)]).map_err(|e| e.to_string())? + // Reject oversized source rasters before the image importer allocates pixels. + let dimensions = image::ImageReader::new(std::io::Cursor::new(&bytes)) + .with_guessed_format() + .map_err(|e| e.to_string())? + .into_dimensions() + .map_err(|e| e.to_string())?; + let decoded_bytes = u64::from(dimensions.0).saturating_mul(u64::from(dimensions.1)).saturating_mul(4); + if decoded_bytes > budget { + return Err("source image exceeds remaining raster memory budget; reduce image dimensions or split the batch".into()); + } + Session::new().with_timestamp(timestamp).create_from_images(&[(name.to_string(), bytes)]).map_err(|e| e.to_string())? }; if task.analysis_only { let selected = if task.preprocessing.pages.trim().is_empty() { pdf } else { // COS-only page selection; do not open an interactive document or run its scripts. - Session::new().combine_ranges(&[(name.to_string(), pdf, Some(task.preprocessing.pages.clone()))]).map_err(|e| e.to_string())? + Session::new() + .with_timestamp(timestamp) + .combine_ranges(&[(name.to_string(), pdf, Some(task.preprocessing.pages.clone()))]) + .map_err(|e| e.to_string())? }; return Ok(preprocess::BlankResult { document: Some(selected), blank_pages: None }); } - let pdf = preprocess::prepare(pdf, &task.preprocessing, f64::from(task.settings.dpi), should_stop)?; + let pdf = preprocess::prepare_bounded(pdf, &task.preprocessing, f64::from(task.settings.dpi), budget, timestamp, should_stop)?; let split = if task.remove_blank_pages { - preprocess::separate_blank_pages(pdf, should_stop)? + preprocess::separate_blank_pages_at(pdf, timestamp, should_stop)? } else { preprocess::BlankResult { document: Some(pdf), blank_pages: None } }; @@ -1005,7 +1269,7 @@ fn prepare( } let model = ocr::engine()?; let mut cancelled = false; - let mut session = Session::new(); + let mut session = Session::new().with_timestamp(timestamp); let id = session.open(name.as_ref(), None, pdf, None).map_err(|e| e.to_string())?; let doc = session.get(id).ok_or("document unavailable")?; if let Some(reason) = &doc.read_only_reason { @@ -1145,12 +1409,14 @@ fn reset_for_run(config_path: &Path, task_id: &str, root: Option<&Path>, immedia let lock = OpenOptions::new().create(true).truncate(false).read(true).write(true).open(lock_path).map_err(|e| e.to_string())?; lock.try_lock().map_err(|_| "worker is busy; stop the task and retry shortly".to_string())?; let mut state = read_state(&path)?; - state.tasks.remove(task_id); - if immediate { - state.tasks.insert(task_id.into(), TaskStatus { batch_active: true, ..Default::default() }); + let publication = state.tasks.remove(task_id).and_then(|status| status.attempt_started_at); + if immediate || publication.is_some() { + state.tasks.insert(task_id.into(), TaskStatus { batch_active: immediate, attempt_started_at: publication, ..Default::default() }); } state.observations.retain(|k, _| !k.starts_with(&format!("{task_id}:"))); - save_json(&confined(&state_path(&path), root)?, &state) + let epoch = state.observation_epochs.entry(task_id.into()).or_default(); + *epoch = epoch.checked_add(1).ok_or("too many explicit task resets")?; + journal::save(&path, &mut state) } #[cfg(test)] @@ -1342,7 +1608,9 @@ mod tests { }, ); let observation_key = format!("{}:{}", task.id, confined(&path, None).unwrap().display()); - state.observations.insert(observation_key, Observation { fingerprint: fingerprint(&path).unwrap(), since: 1, attempted: true }); + state + .observations + .insert(observation_key, Observation { fingerprint: fingerprint(&path).unwrap(), since: 1, attempted: true, receipt_key: None }); settled(&task, &mut state, &config).unwrap(); assert!(!path.exists()); assert_eq!(fs::read_dir(&task.archive).unwrap().count(), 1); @@ -1376,7 +1644,7 @@ mod tests { cleanup: Some(PendingLocalAction::Delete), }, ); - state.observations.insert(key, Observation { fingerprint: stamp, since: 1, attempted: true }); + state.observations.insert(key, Observation { fingerprint: stamp, since: 1, attempted: true, receipt_key: None }); match corruption { "missing" => fs::remove_file(&output).unwrap(), "replaced" => fs::write(&output, b"a different result").unwrap(), @@ -1574,3 +1842,7 @@ mod tests { assert_eq!(remove_blank_pages(text.clone()).unwrap(), text); } } + +#[cfg(test)] +#[path = "hot_folder_server_tests.rs"] +mod server_tests; diff --git a/crates/engine/src/hot_folder_journal.rs b/crates/engine/src/hot_folder_journal.rs new file mode 100644 index 0000000000..d8b57566cb --- /dev/null +++ b/crates/engine/src/hot_folder_journal.rs @@ -0,0 +1,184 @@ +//! Small active journal plus durable, sharded history. Historical receipts are never +//! discarded to make room: an evicted source version must remain deduplicated after restart. +//! Individual reads/writes and the hot cache are bounded; disk history grows with unique work. +use super::*; + +const HOT_ENTRIES: usize = 256; +const HOT_BYTES: usize = 2 * 1024 * 1024; + +#[derive(Serialize, Deserialize)] +struct Entry { + key: String, + value: T, +} + +fn is_link(meta: &fs::Metadata) -> bool { + #[cfg(windows)] + { + use std::os::windows::fs::MetadataExt; + meta.file_attributes() & 0x400 != 0 + } + #[cfg(not(windows))] + meta.file_type().is_symlink() +} + +fn directory(path: &Path, create: bool) -> Result { + match fs::symlink_metadata(path) { + Ok(meta) if meta.is_dir() && !is_link(&meta) => Ok(true), + Ok(_) => Err("hot folder history directory must not be a link or file".into()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound && !create => Ok(false), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + #[cfg(not(unix))] + let _ = &mut builder; + builder.create(path).map_err(|e| e.to_string())?; + #[cfg(unix)] + if rustix::process::geteuid().as_raw() == 0 { + use std::os::unix::fs::MetadataExt; + let owner = fs::metadata(path.parent().ok_or("history needs a parent")?).map_err(|e| e.to_string())?; + std::os::unix::fs::chown(path, Some(owner.uid()), Some(owner.gid())).map_err(|e| e.to_string())?; + } + sync_parent(path)?; + Ok(true) + } + Err(e) => Err(e.to_string()), + } +} + +fn location(config: &Path, kind: &str, key: &str, create: bool) -> Result, String> { + let digest: String = Sha256::digest(key.as_bytes()).iter().map(|byte| format!("{byte:02x}")).collect(); + let base = config.with_extension("journal"); + let group = base.join(kind); + let shard = group.join(digest.get(..2).ok_or("invalid history key")?); + for path in [&base, &group, &shard] { + if !directory(path, create)? { + return Ok(None); + } + } + let path = shard.join(format!("{digest}.json")); + match fs::symlink_metadata(&path) { + Ok(meta) if is_link(&meta) || !meta.is_file() => Err("hot folder history must be a regular file".into()), + Ok(_) => Ok(Some(path)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(create.then_some(path)), + Err(e) => Err(e.to_string()), + } +} + +pub(crate) fn read Deserialize<'de>>(config: &Path, kind: &str, key: &str) -> Result, String> { + let Some(path) = location(config, kind, key, false)? else { return Ok(None) }; + let entry: Entry = read_json(&path)?; + if entry.key != key { + return Err("hot folder history key does not match its durable receipt".into()); + } + Ok(Some(entry.value)) +} + +pub(crate) fn write(config: &Path, kind: &str, key: &str, value: &T) -> Result<(), String> { + let path = location(config, kind, key, true)?.ok_or("history path unavailable")?; + save_json(&path, &Entry { key: key.to_owned(), value }) +} + +/// A full write-ahead snapshot makes every group receipt visible together even if +/// the process is killed halfway through sharding its history. +pub(crate) fn pending_path(target: &Path) -> PathBuf { + target.with_extension("pending.json") +} + +pub(crate) fn pending Deserialize<'de>>(target: &Path) -> Result, String> { + let path = pending_path(target); + match fs::symlink_metadata(&path) { + Ok(meta) if is_link(&meta) || !meta.is_file() => Err("pending journal must be a regular file".into()), + Ok(_) => read_json(&path).map(Some), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.to_string()), + } +} + +pub(crate) fn begin(target: &Path, value: &T) -> Result<(), String> { + save_json(&pending_path(target), value) +} + +pub(crate) fn finish(target: &Path) -> Result<(), String> { + let path = pending_path(target); + match fs::remove_file(&path) { + Ok(()) => sync_parent(&path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.to_string()), + } +} + +pub(super) fn receipt(config: &Path, state: &State, key: &str) -> Result, String> { + match state.receipts.get(key) { + Some(value) => Ok(Some(value.clone())), + None => read(config, "receipts", key), + } +} + +fn observation_key(state: &State, key: &str) -> String { + let task = key.split(':').next().unwrap_or_default(); + format!("{}:{key}", state.observation_epochs.get(task).copied().unwrap_or_default()) +} + +pub(super) fn observation(config: &Path, state: &State, key: &str) -> Result, String> { + match state.observations.get(key) { + Some(value) => Ok(Some(value.clone())), + None => read(config, "observations", &observation_key(state, key)), + } +} + +/// Publication order is intentional: archive entries first, then the compact active state. +/// A killed process sees either the old complete cache or the new cache plus durable history. +pub(super) fn save(config: &Path, state: &mut State) -> Result<(), String> { + save_with(config, state, &|_| Ok(())) +} + +pub(super) fn save_with(config: &Path, state: &mut State, after_spill: &dyn Fn(usize) -> Result<(), String>) -> Result<(), String> { + let links: BTreeMap<_, _> = state + .receipts + .iter() + .filter_map(|(key, receipt)| key.rsplit_once(':').map(|(source, _)| ((source.to_owned(), receipt.source_stamp.clone()), key.clone()))) + .collect(); + for (key, observation) in &mut state.observations { + if observation.receipt_key.is_none() { + observation.receipt_key = links.get(&(key.clone(), observation.fingerprint.clone())).cloned(); + } + } + let target = state_path(config); + let needs_spill = state.receipts.len() > HOT_ENTRIES + || state.observations.len() > HOT_ENTRIES + || serde_json::to_vec(&state.receipts).map_err(|e| e.to_string())?.len() > HOT_BYTES + || serde_json::to_vec(&state.observations).map_err(|e| e.to_string())?.len() > HOT_BYTES; + if needs_spill { + begin(&target, state)?; + } + let mut spilled = 0; + while state.receipts.len() > HOT_ENTRIES || serde_json::to_vec(&state.receipts).map_err(|e| e.to_string())?.len() > HOT_BYTES { + let Some((key, value)) = state.receipts.first_key_value() else { break }; + write(config, "receipts", key, value)?; + let key = key.clone(); + state.receipts.remove(&key); + spilled += 1; + after_spill(spilled)?; + } + while state.observations.len() > HOT_ENTRIES || serde_json::to_vec(&state.observations).map_err(|e| e.to_string())?.len() > HOT_BYTES { + let Some((key, value)) = state.observations.first_key_value() else { break }; + write(config, "observations", &observation_key(state, key), value)?; + let key = key.clone(); + state.observations.remove(&key); + spilled += 1; + after_spill(spilled)?; + } + if state.log.len() > 1000 { + state.log.drain(..state.log.len().saturating_sub(1000)); + } + while !state.log.is_empty() && serde_json::to_vec(&state.log).map_err(|e| e.to_string())?.len() > 1024 * 1024 { + state.log.drain(..state.log.len().div_ceil(2)); + } + save_json(&target, state)?; + finish(&target) +} diff --git a/crates/engine/src/hot_folder_output.rs b/crates/engine/src/hot_folder_output.rs index 3f74aef48f..504213c33c 100644 --- a/crates/engine/src/hot_folder_output.rs +++ b/crates/engine/src/hot_folder_output.rs @@ -157,6 +157,17 @@ impl OutputPlan { /// Validate before reading source files. The caller also checks destinations against /// its full set of input folders to prevent recursive output/input loops. pub fn validate(&self, default_output: &Path, root: Option<&Path>) -> Result<(), String> { + self.validate_settings()?; + for step in &self.steps { + let folder = confined(step.folder.as_deref().unwrap_or(default_output), root)?; + if !folder.is_dir() { + return Err(format!("output folder does not exist: {}", folder.display())); + } + } + Ok(()) + } + + pub fn validate_settings(&self) -> Result<(), String> { if self.steps.is_empty() || self.steps.len() > 16 { return Err("choose between 1 and 16 output saving steps".into()); } @@ -168,10 +179,6 @@ impl OutputPlan { if step.language.is_empty() || step.language.len() > 63 || !step.language.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') { return Err("EPUB language must be a language tag such as de-AT or und".into()); } - let folder = confined(step.folder.as_deref().unwrap_or(default_output), root)?; - if !folder.is_dir() { - return Err(format!("output folder does not exist: {}", folder.display())); - } } Ok(()) } @@ -203,6 +210,9 @@ pub struct OutputProof { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct OutputError { pub message: String, + /// A filesystem publication/staging failure can recover without changing the input. + #[serde(default)] + pub retryable: bool, /// Complete files already published (or reused after byte-identical comparison). /// No source action is permitted after this error, even when this list is nonempty. pub outputs: Vec, @@ -227,13 +237,14 @@ pub fn save( should_stop: &dyn Fn() -> bool, ) -> Result { let mut receipt = OutputReceipt::default(); + let mut retryable = false; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - save_inner(inputs, default_output, plan, task_name, batch_time, root, should_stop, &mut receipt) + save_inner(inputs, default_output, plan, task_name, batch_time, root, should_stop, &mut receipt, &mut retryable) })); match result { Ok(Ok(())) => Ok(receipt), - Ok(Err(message)) => Err(OutputError { message, outputs: receipt.outputs }), - Err(_) => Err(OutputError { message: "output conversion failed unexpectedly".into(), outputs: receipt.outputs }), + Ok(Err(message)) => Err(OutputError { message, outputs: receipt.outputs, retryable }), + Err(_) => Err(OutputError { message: "output conversion failed unexpectedly".into(), outputs: receipt.outputs, retryable: false }), } } @@ -298,8 +309,11 @@ fn save_inner( root: Option<&Path>, should_stop: &dyn Fn() -> bool, receipt: &mut OutputReceipt, + retryable: &mut bool, ) -> Result<(), String> { + *retryable = true; plan.validate(default_output, root)?; + *retryable = false; if inputs.is_empty() || inputs.len() > 10_000 || inputs.iter().any(|input| !relative_ok(&input.relative)) { return Err("output batch needs 1..10000 inputs with relative paths containing normal components only".into()); } @@ -314,24 +328,28 @@ fn save_inner( let mut staged = Vec::new(); let mut staged_bytes = 0_u64; for step in &plan.steps { + *retryable = true; let base = confined(step.folder.as_deref().unwrap_or(default_output), root)?; + *retryable = false; for (index, group) in groups(inputs, step.grouping, task_name).into_iter().enumerate() { stopped(should_stop)?; + *retryable = true; let folder = safe_subfolder(&base, &group.subfolder)?; + *retryable = false; let stem = expand_name(&step.name_template, &group.name, &group.folder_name, task_name, timestamp, index.saturating_add(1))?; let bytes = if let [input] = group.inputs.as_slice() { input.bytes.clone() } else { let sources = group.inputs.iter().map(|input| (input.relative.to_string_lossy().into_owned(), input.bytes.clone())).collect::>(); - Session::new().combine(&sources).map_err(|error| error.to_string())? + Session::new().with_timestamp(timestamp.timestamp()).combine(&sources).map_err(|error| error.to_string())? }; if step.format == Format::Analysis { let report = crate::hot_folder_analysis::analyze(bytes.as_slice(), &[], should_stop)?; - stage(&mut staged, &mut staged_bytes, &folder, &stem, "json", &report)?; + stage(&mut staged, &mut staged_bytes, &folder, &stem, "json", &report, retryable)?; continue; } - let mut session = Session::new(); + let mut session = Session::new().with_timestamp(timestamp.timestamp()); let id = session.open(format!("{stem}.pdf"), None, bytes.clone(), None).map_err(|error| error.to_string())?; let document = session.get(id).ok_or("output document unavailable")?; let pages = document.info.pages.len(); @@ -351,6 +369,7 @@ fn save_inner( &format!("{stem}_page_{:04}", page.saturating_add(1)), step.format.extension(), &bytes, + retryable, )?; } } else { @@ -359,6 +378,7 @@ fn save_inner( Format::ImagePdf => { let mut sources = Vec::new(); let mut images_size = 0_u64; + let mut prepared_size = 0_u64; for page in 0..pages { stopped(should_stop)?; warn_resolution(&exporter, page, step.dpi, &stem, receipt); @@ -374,6 +394,10 @@ fn save_inner( crate::ImageResolution::Dpi(exporter.dpi_used(page, step.dpi)), ) .map_err(|error| error.to_string())?; + prepared_size = prepared_size.saturating_add(pdf.len() as u64); + if prepared_size > MAX_INPUT_BYTES { + return Err("prepared image-only PDF exceeds 512 MiB; lower the resolution".into()); + } sources.push((format!("page-{}.pdf", page.saturating_add(1)), pdf)); } session.combine(&sources).map_err(|error| error.to_string())? @@ -386,7 +410,7 @@ fn save_inner( Format::Epub => Arc::new(pdfcraft_export::epub(&document.export_pages(), &stem, &step.language)?), Format::Png | Format::Jpeg | Format::Tiff | Format::Analysis => return Err("invalid output state".into()), }; - stage(&mut staged, &mut staged_bytes, &folder, &stem, step.format.extension(), output.as_slice())?; + stage(&mut staged, &mut staged_bytes, &folder, &stem, step.format.extension(), output.as_slice(), retryable)?; } } } @@ -395,7 +419,9 @@ fn save_inner( stopped(should_stop)?; let bytes = file.size; let sha256 = file.sha256.clone(); + *retryable = true; let path = publish(file, root)?; + *retryable = false; receipt.proofs.push(OutputProof { path: path.clone(), bytes, sha256 }); receipt.outputs.push(path); } @@ -420,13 +446,23 @@ fn digest(bytes: &[u8]) -> String { Sha256::digest(bytes).iter().map(|byte| format!("{byte:02x}")).collect() } -fn stage(staged: &mut Vec, total: &mut u64, folder: &Path, stem: &str, extension: &'static str, bytes: &[u8]) -> Result<(), String> { +fn stage( + staged: &mut Vec, + total: &mut u64, + folder: &Path, + stem: &str, + extension: &'static str, + bytes: &[u8], + retryable: &mut bool, +) -> Result<(), String> { *total = total.saturating_add(bytes.len() as u64); if bytes.is_empty() || *total > MAX_STAGED_BYTES || staged.len() >= MAX_OUTPUT_FILES { return Err("output batch exceeds 1 GiB or 10000 files, or an exporter returned an empty file; use smaller batches".into()); } + *retryable = true; let mut file = tempfile::Builder::new().prefix(".pdfcraft-output-").tempfile_in(folder).map_err(|error| error.to_string())?; file.write_all(bytes).and_then(|_| file.as_file().sync_all()).map_err(|error| format!("cannot stage output: {error}"))?; + *retryable = false; staged.push(Staged { file, folder: folder.to_owned(), stem: stem.to_owned(), extension, size: bytes.len() as u64, sha256: digest(bytes) }); Ok(()) } diff --git a/crates/engine/src/hot_folder_preprocess.rs b/crates/engine/src/hot_folder_preprocess.rs index cc2fabc17a..86101955bd 100644 --- a/crates/engine/src/hot_folder_preprocess.rs +++ b/crates/engine/src/hot_folder_preprocess.rs @@ -97,11 +97,25 @@ pub fn parse_pages(value: &str, count: Option) -> Result, Stri } pub fn prepare(bytes: Arc>, settings: &Settings, dpi: f64, should_stop: &dyn Fn() -> bool) -> Result>, String> { + prepare_bounded(bytes, settings, dpi, 512 * 1024 * 1024, Session::new().now_secs(), should_stop) +} + +pub(crate) fn prepare_bounded( + bytes: Arc>, + settings: &Settings, + dpi: f64, + budget: u64, + timestamp: i64, + should_stop: &dyn Fn() -> bool, +) -> Result>, String> { settings.validate()?; + if bytes.len() as u64 > budget { + return Err("prepared input exceeds remaining batch memory budget; use smaller batches".into()); + } if !settings.changes_images() && settings.pages.trim().is_empty() { return Ok(bytes); } - let mut session = Session::new(); + let mut session = Session::new().with_timestamp(timestamp); let id = session.open("input.pdf", None, bytes, None).map_err(|e| e.to_string())?; let doc = session.get(id).ok_or("document unavailable")?; if let Some(reason) = &doc.read_only_reason { @@ -117,7 +131,7 @@ pub fn prepare(bytes: Arc>, settings: &Settings, dpi: f64, should_stop: } let mut exporter = crate::export::Exporter::new(doc); let mut sources = Vec::new(); - let mut total = 0usize; + let mut total = 0_u64; for page in selected { if should_stop() { return Err("task stopped during image preprocessing".into()); @@ -137,8 +151,7 @@ pub fn prepare(bytes: Arc>, settings: &Settings, dpi: f64, should_stop: let image = image::imageops::crop_imm(&rgb, left, 0, width, rgb.height()).to_image(); let mut encoded = Cursor::new(Vec::new()); image::DynamicImage::ImageRgb8(image).write_to(&mut encoded, image::ImageFormat::Png).map_err(|e| e.to_string())?; - total = total.saturating_add(encoded.get_ref().len()); - if total > 512 * 1024 * 1024 { + if total.saturating_add(encoded.get_ref().len() as u64) > budget { return Err("prepared images exceed 512 MiB; reduce resolution or select fewer pages".into()); } let name = format!("page-{}-{left}.png", page.saturating_add(1)); @@ -148,10 +161,18 @@ pub fn prepare(bytes: Arc>, settings: &Settings, dpi: f64, should_stop: crate::ImageResolution::Dpi(exporter.dpi_used(page, dpi)), ) .map_err(|e| e.to_string())?; + total = total.saturating_add(pdf.len() as u64); + if total > budget { + return Err("prepared PDFs exceed remaining batch memory budget; use smaller batches".into()); + } sources.push((name, pdf)); } } - session.combine(&sources).map_err(|e| e.to_string()) + let combined = session.combine(&sources).map_err(|e| e.to_string())?; + if combined.len() as u64 > budget { + return Err("combined prepared PDF exceeds remaining batch memory budget".into()); + } + Ok(combined) } fn transform(image: &mut image::RgbImage, settings: &Settings) { @@ -224,7 +245,11 @@ pub struct BlankResult { } /// Strict white-only detection retains faint marks, existing text and interactive content. pub fn separate_blank_pages(bytes: Arc>, should_stop: &dyn Fn() -> bool) -> Result { - let mut session = Session::new(); + separate_blank_pages_at(bytes, Session::new().now_secs(), should_stop) +} + +pub(crate) fn separate_blank_pages_at(bytes: Arc>, timestamp: i64, should_stop: &dyn Fn() -> bool) -> Result { + let mut session = Session::new().with_timestamp(timestamp); let id = session.open("scan.pdf", None, bytes.clone(), None).map_err(|e| e.to_string())?; let doc = session.get(id).ok_or("document unavailable")?; if !doc.info.annotations.is_empty() || !doc.info.fields.is_empty() || !doc.info.links.is_empty() { diff --git a/crates/engine/src/hot_folder_server_tests.rs b/crates/engine/src/hot_folder_server_tests.rs new file mode 100644 index 0000000000..e55149495c --- /dev/null +++ b/crates/engine/src/hot_folder_server_tests.rs @@ -0,0 +1,374 @@ +use super::*; + +fn task(base: &Path, id: &str) -> Task { + let input = base.join(format!("{id}-in")); + let output = base.join(format!("{id}-out")); + fs::create_dir(&input).unwrap(); + fs::create_dir(&output).unwrap(); + Task { + id: id.into(), + name: id.into(), + input, + output, + enabled: true, + interval_seconds: 1, + settle_seconds: 1, + settings: ocr::OcrSettings { recognize: false, auto_rotate: false, deskew: false, ..Default::default() }, + ..Default::default() + } +} + +fn pdf() -> Arc> { + Session::new().create_from_text("server", "An original survives a failed output").unwrap() +} + +fn make_due(config: &Path) { + let mut state = read_state(config).unwrap(); + for status in state.tasks.values_mut() { + status.next_check = 0; + } + for observation in state.observations.values_mut() { + observation.since = 0; + } + journal::save(config, &mut state).unwrap(); +} + +#[test] +fn offline_and_disabled_sources_cannot_stop_other_tasks_and_recover_automatically() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let remote = task(base.path(), "remote"); + let local = task(base.path(), "local"); + let mut disabled = task(base.path(), "disabled"); + disabled.enabled = false; + fs::remove_dir(&remote.input).unwrap(); + fs::remove_dir(&disabled.input).unwrap(); + fs::remove_dir(&disabled.output).unwrap(); + let settings = Config { tasks: vec![remote.clone(), disabled, local.clone()] }; + save_json(&config, &settings).unwrap(); + validate_structure(&settings, None).unwrap(); + initialize_config(&config).unwrap(); + fs::write(local.input.join("local.pdf"), pdf().as_slice()).unwrap(); + poll(&config, None).unwrap(); + make_due(&config); + let state = poll(&config, None).unwrap(); + assert_eq!(state.tasks["local"].completed, 1); + assert!(local.output.join("local.pdf").exists()); + assert!(state.tasks["remote"].retry_count > 0); + assert!(!state.tasks["remote"].blocked); + fs::create_dir(&remote.input).unwrap(); + fs::write(remote.input.join("remote.pdf"), pdf().as_slice()).unwrap(); + make_due(&config); + poll_task_cancelable(&config, None, "remote", &|| false).unwrap(); + make_due(&config); + let state = poll_task_cancelable(&config, None, "remote", &|| false).unwrap(); + assert_eq!(state.tasks["remote"].completed, 1); + assert_eq!(state.tasks["remote"].retry_count, 0); + assert_eq!(state.tasks["local"].completed, 1); +} + +#[test] +fn transient_publication_failure_retries_the_whole_group_without_skipping_sources() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let mut task = task(base.path(), "group"); + task.originals = Originals::Delete; + task.output_plan.steps[0].grouping = output::Grouping::All; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + fs::write(task.input.join("a.pdf"), pdf().as_slice()).unwrap(); + fs::write(task.input.join("b.pdf"), pdf().as_slice()).unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + let moved = base.path().join("temporarily-unavailable"); + let interrupt = std::cell::Cell::new(false); + let outage = || { + if !interrupt.replace(true) { + fs::rename(&task.output, &moved).unwrap(); + fs::write(&task.output, b"mount unavailable").unwrap(); + } + false + }; + let error = step(&task, &mut state, 13, &config, None, &outage).unwrap_err(); + assert!(error.contains("all originals retained")); + assert!(state.observations.values().all(|observation| !observation.attempted)); + assert!(!state.tasks["group"].blocked); + task_failure(&mut state, "group", error, now()); + journal::save(&config, &mut state).unwrap(); + fs::remove_file(&task.output).unwrap(); + fs::rename(&moved, &task.output).unwrap(); + make_due(&config); + let state = poll(&config, None).unwrap(); + assert_eq!(state.tasks["group"].completed, 2); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 1); + assert_eq!(fs::read_dir(&task.input).unwrap().count(), 0); +} + +#[test] +fn permanent_bad_document_is_skipped_per_file_but_never_dropped_from_a_group() { + for grouped in [false, true] { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let mut task = task(base.path(), "documents"); + if grouped { + task.output_plan.steps[0].grouping = output::Grouping::All; + } + task.output_plan.steps[0].format = output::Format::Txt; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + fs::write(task.input.join("a-bad.pdf"), b"malformed PDF").unwrap(); + fs::write(task.input.join("b-good.pdf"), pdf().as_slice()).unwrap(); + poll(&config, None).unwrap(); + make_due(&config); + let failed = poll(&config, None).unwrap(); + assert_eq!(failed.tasks["documents"].blocked, grouped); + make_due(&config); + let retried = poll(&config, None).unwrap(); + assert_eq!(retried.tasks["documents"].completed, if grouped { 0 } else { 1 }); + assert!(task.input.join("a-bad.pdf").exists()); + } +} + +#[test] +fn lifetime_receipts_are_archived_without_losing_deduplication_or_cleanup_proofs() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let task = task(base.path(), "live"); + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let source = task.input.join("first.pdf"); + fs::write(&source, pdf().as_slice()).unwrap(); + let mut state = State::default(); + step(&task, &mut state, 10, &config, None, &|| false).unwrap(); + step(&task, &mut state, 13, &config, None, &|| false).unwrap(); + let original = state.receipts.first_key_value().unwrap().1.clone(); + let bytes = fs::read(&source).unwrap(); + let mut first_key = None; + // Originals=Keep is the default: finished physical inputs must not exhaust the + // enumerator either. Every historical entry below has a real retained source. + for index in 0..10_001 { + let retained = task.input.join(format!("old-{index:05}.pdf")); + fs::write(&retained, &bytes).unwrap(); + let stamp = fingerprint(&retained).unwrap(); + let key = format!("{}:{}", task.id, confined(&retained, None).unwrap().display()); + let receipt_key = format!("{key}:{}", original.source_hash); + first_key.get_or_insert_with(|| receipt_key.clone()); + let mut receipt = original.clone(); + receipt.source_stamp = stamp.clone(); + state.receipts.insert(receipt_key.clone(), receipt); + state.observations.insert(key, Observation { fingerprint: stamp, since: 1, attempted: true, receipt_key: Some(receipt_key) }); + if index % 256 == 255 { + journal::save(&config, &mut state).unwrap(); + } + } + journal::save(&config, &mut state).unwrap(); + let mut restored = read_state(&config).unwrap(); + assert!(restored.receipts.len() <= 256); + assert!(fs::metadata(state_path(&config)).unwrap().len() < 3 * 1024 * 1024); + let first_key = first_key.unwrap(); + let old = journal::receipt(&config, &restored, &first_key).unwrap().unwrap(); + verify_outputs(&old, None).unwrap(); + step(&task, &mut restored, 20, &config, None, &|| false).unwrap(); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 1); + fs::write(task.input.join("next.pdf"), pdf().as_slice()).unwrap(); + step(&task, &mut restored, 26, &config, None, &|| false).unwrap(); + step(&task, &mut restored, 29, &config, None, &|| false).unwrap(); + assert!(task.output.join("next.pdf").exists()); + // An explicit retry clears observations, never the archived content receipts. + reset(&config, "live", None).unwrap(); + restored = read_state(&config).unwrap(); + verify_outputs(&journal::receipt(&config, &restored, &first_key).unwrap().unwrap(), None).unwrap(); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 2); +} + +#[test] +fn child_failures_use_capped_backoff_and_do_not_change_output_receipts() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let task = task(base.path(), "child"); + save_config(&config, &Config { tasks: vec![task] }, None).unwrap(); + for attempt in 1..=20 { + let before = now(); + let state = record_task_failure(&config, None, "child", "child deadline expired").unwrap(); + assert_eq!(state.tasks["child"].retry_count, attempt); + assert!(state.tasks["child"].next_check >= before + 5); + assert!(state.tasks["child"].next_check <= now() + 900); + assert!(!state.tasks["child"].blocked); + assert!(state.receipts.is_empty()); + } +} + +#[test] +fn a_retried_publication_reuses_its_durable_timestamp_and_idle_polls_do_not_rewrite_state() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let mut task = task(base.path(), "identity"); + task.output_plan.steps[0].name_template = "{name}-{time}".into(); + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + fs::write(task.input.join("document.pdf"), pdf().as_slice()).unwrap(); + let mut state = poll(&config, None).unwrap(); + state.tasks.get_mut("identity").unwrap().attempt_started_at = Some(1234); + journal::save(&config, &mut state).unwrap(); + output::save( + &[output::Input { relative: "document.pdf".into(), bytes: Arc::new(fs::read(task.input.join("document.pdf")).unwrap()) }], + &task.output, + &task.output_plan, + &task.name, + 1234, + None, + &|| false, + ) + .unwrap(); + record_task_failure(&config, None, "identity", "child interrupted after publication").unwrap(); + let before = fs::read(state_path(&config)).unwrap(); + let state = poll(&config, None).unwrap(); + assert_eq!(state.tasks["identity"].attempt_started_at, Some(1234)); + assert_eq!(fs::read(state_path(&config)).unwrap(), before); + reset(&config, "identity", None).unwrap(); + assert_eq!(read_state(&config).unwrap().tasks["identity"].attempt_started_at, Some(1234)); + poll(&config, None).unwrap(); + make_due(&config); + let resumed = poll(&config, None).unwrap(); + assert_eq!(resumed.tasks["identity"].completed, 1); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 1); +} + +#[test] +fn expanded_preprocessing_is_rejected_before_exceeding_the_remaining_batch_budget() { + let bytes = pdf(); + let settings = preprocess::Settings { grayscale: true, ..Default::default() }; + let error = preprocess::prepare_bounded(bytes.clone(), &settings, 72.0, (bytes.len() - 1) as u64, 1234, &|| false).unwrap_err(); + assert!(error.contains("budget")); + let error = preprocess::prepare_bounded(bytes.clone(), &settings, 300.0, bytes.len() as u64 + 128, 1234, &|| false).unwrap_err(); + assert!(error.contains("prepared")); + let base = tempfile::tempdir().unwrap(); + let task = task(base.path(), "raster"); + let mut image = std::io::Cursor::new(Vec::new()); + image::DynamicImage::new_rgb8(20, 20).write_to(&mut image, image::ImageFormat::Png).unwrap(); + let error = prepare(&task, Path::new("scan.png"), image.into_inner(), &base.path().join("tasks.json"), 1000, 1234, &|| false).err().unwrap(); + assert!(error.contains("raster memory budget")); +} + +#[test] +fn interrupted_group_spill_recovers_every_receipt_as_one_transaction() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let mut task = task(base.path(), "aa-group"); + task.output_plan.steps[0].grouping = output::Grouping::All; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let bytes = pdf(); + for name in ["one.pdf", "two.pdf"] { + fs::write(task.input.join(name), bytes.as_slice()).unwrap(); + } + let inputs = ["one.pdf", "two.pdf"].map(|relative| output::Input { relative: relative.into(), bytes: bytes.clone() }); + let output = output::save(&inputs, &task.output, &task.output_plan, &task.name, 12, None, &|| false).unwrap(); + let hash: String = Sha256::digest(bytes.as_slice()).iter().map(|byte| format!("{byte:02x}")).collect(); + let mut state = State::default(); + journal::save(&config, &mut state).unwrap(); + for name in ["one.pdf", "two.pdf"] { + let source = task.input.join(name); + state.receipts.insert( + format!("{}:{}:{hash}", task.id, confined(&source, None).unwrap().display()), + Receipt { + outputs: output.outputs.clone(), + source_hash: hash.clone(), + source_stamp: fingerprint(&source).unwrap(), + output_proofs: output.proofs.clone(), + cleanup: None, + }, + ); + } + let receipt = state.receipts.first_key_value().unwrap().1.clone(); + for index in 0..256 { + state.receipts.insert(format!("zz-completed:{index}"), receipt.clone()); + } + assert!( + journal::save_with(&config, &mut state, &|count| if count == 1 { Err("simulated crash after one spill".into()) } else { Ok(()) }).is_err() + ); + assert!(journal::pending_path(&state_path(&config)).exists()); + let mut restored = read_state(&config).unwrap(); + assert_eq!(restored.receipts.len(), 258); + step(&task, &mut restored, 20, &config, None, &|| false).unwrap(); + step(&task, &mut restored, 23, &config, None, &|| false).unwrap(); + journal::save(&config, &mut restored).unwrap(); + assert!(!journal::pending_path(&state_path(&config)).exists()); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 1); + assert!(task.input.join("one.pdf").exists()); + assert!(task.input.join("two.pdf").exists()); +} + +#[test] +fn changing_a_project_manifest_cannot_consume_another_tasks_originals() { + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let mut project = task(base.path(), "project"); + let local = task(base.path(), "local"); + let manifest = base.path().join("project.json"); + let original = base.path().join("original.pdf"); + fs::write(&original, pdf().as_slice()).unwrap(); + let mut contents = sources::ProjectManifest { files: vec!["original.pdf".into()], ..Default::default() }; + sources::write_project_manifest(&manifest, &contents, None).unwrap(); + project.source = sources::SourceConfig::Project(sources::ProjectSource { manifest: manifest.clone(), action: sources::SourceAction::Delete }); + save_config(&config, &Config { tasks: vec![project, local.clone()] }, None).unwrap(); + let protected = local.input.join("unprocessed.pdf"); + fs::write(&protected, pdf().as_slice()).unwrap(); + contents.files.push("local-in/unprocessed.pdf".into()); + save_json(&manifest, &contents).unwrap(); + let result = poll_task_cancelable(&config, None, "project", &|| false).unwrap(); + assert!(result.tasks["project"].error.as_ref().unwrap().contains("overlap")); + assert!(protected.exists()); + assert!(original.exists()); + assert!(!result.tasks.contains_key("local")); +} + +#[test] +fn replayed_scan_and_group_exports_keep_the_transaction_modification_date() { + if !ocr::available() { + return; + } + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let mut task = task(base.path(), "clock"); + task.settings.recognize = true; + task.preprocessing.grayscale = true; + task.output_plan.steps[0].grouping = output::Grouping::All; + save_config(&config, &Config { tasks: vec![task.clone()] }, None).unwrap(); + let mut session = Session::new(); + let id = session.open("source", None, pdf(), None).unwrap(); + let png = crate::export::Exporter::new(session.get(id).unwrap()).png(0, 150.0).unwrap(); + let mut results = Vec::new(); + for _ in 0..2 { + let prepared = prepare(&task, Path::new("scan.png"), png.clone(), &config, MAX_INPUT, 1234, &|| false).ok().unwrap().document.unwrap(); + assert_transaction_date(prepared.clone(), 1234); + results.push(prepared); + } + assert_eq!(results[0], results[1]); + let inputs: Vec<_> = + results.into_iter().enumerate().map(|(i, bytes)| output::Input { relative: format!("scan-{i}.pdf").into(), bytes }).collect(); + let first = output::save(&inputs, &task.output, &task.output_plan, &task.name, 1234, None, &|| false).unwrap(); + let second = output::save(&inputs, &task.output, &task.output_plan, &task.name, 1234, None, &|| false).unwrap(); + assert_eq!(first.outputs, second.outputs); + assert_eq!(fs::read_dir(&task.output).unwrap().count(), 1); + assert_transaction_date(Arc::new(fs::read(&first.outputs[0]).unwrap()), 1234); +} + +fn assert_transaction_date(bytes: Arc>, timestamp: i64) { + let document = pdfcraft_cos::Document::open(bytes).unwrap(); + let info = document.dict(document.trailer().get(b"Info").unwrap()).unwrap(); + assert_eq!(info.get(b"ModDate").unwrap().as_string().unwrap().to_text(), pdfcraft_cos::pdf_date(timestamp)); +} + +#[test] +fn first_calendar_schedule_is_persisted_once_without_probing_the_input() { + use chrono::Timelike; + let base = tempfile::tempdir().unwrap(); + let config = base.path().join("tasks.json"); + let mut task = task(base.path(), "calendar"); + task.schedule = Schedule::Daily { hour: (chrono::Local::now().hour() + 1) % 24, minute: 0 }; + fs::remove_dir(&task.input).unwrap(); + save_json(&config, &Config { tasks: vec![task] }).unwrap(); + let state = poll(&config, None).unwrap(); + assert!(state.tasks["calendar"].next_check > now()); + assert_eq!(state.tasks["calendar"].failed, 0); + assert_eq!(read_state(&config).unwrap().tasks["calendar"].next_check, state.tasks["calendar"].next_check); + let before = fs::read(state_path(&config)).unwrap(); + poll(&config, None).unwrap(); + assert_eq!(fs::read(state_path(&config)).unwrap(), before); +} diff --git a/crates/engine/src/hot_folder_sources.rs b/crates/engine/src/hot_folder_sources.rs index e0dadae6fc..cc5ec8de60 100644 --- a/crates/engine/src/hot_folder_sources.rs +++ b/crates/engine/src/hot_folder_sources.rs @@ -232,7 +232,7 @@ pub fn stage( let mut source = backend(config, root, &control, store)?; let mut report = stage_with(source.as_mut(), config, &input, &mut journal, &control)?; if let SourceConfig::Project(project) = config { - project::validate_and_order(project, root, &journal, &mut report)?; + project::validate_and_order(project, &input, root, &journal, &mut report)?; } Ok(report) }) @@ -263,6 +263,14 @@ fn stage_with( return Err("source path is unsafe or source exceeds the 512 MiB per-file limit".into()); } let source_id = hash(format!("{}\n{}\n{}\n{}", journal.config_hash, remote.key, remote.revision, remote.size).as_bytes()); + if !journal.records.contains_key(&source_id) + && let Some(record) = archived_record(input, &journal.config_hash, &source_id)? + { + if record.acknowledged { + continue; + } + return Err("source history contains an unfinished archive record".into()); + } if journal.records.get(&source_id).is_some_and(|record| record.acknowledged || record.ready) { continue; } @@ -283,7 +291,7 @@ fn stage_with( } } if journal.records.len() >= MAX_ITEMS { - return Err("source journal reached 10000 versions; archive the task and create a new task".into()); + return Err("source queue has 10000 unfinished versions; finish or resolve pending acknowledgements first".into()); } let name = remote.relative.file_name().and_then(|name| name.to_str()).ok_or("source filename is not valid UTF-8")?; let staged_relative = PathBuf::from(format!("{}-{}", source_id.get(..24).ok_or("invalid source ID")?, safe_filename(name))); @@ -356,6 +364,11 @@ pub fn acknowledge( let mut journal = read_journal(&input, config)?; let control = Control::new(should_stop); control.check()?; + if !journal.records.contains_key(&item.source_id) + && let Some(record) = archived_record(&input, &journal.config_hash, &item.source_id)? + { + journal.records.insert(item.source_id.clone(), record); + } let record = journal.records.get_mut(&item.source_id).ok_or("source receipt is absent from the durable journal")?; if record.source != *item || confirmed_sha256 != record.source.sha256 { return Err("output receipt does not match this source version; original retained".into()); @@ -368,7 +381,7 @@ pub fn acknowledge( } record.ready = true; let group = record.remote.group.clone(); - write_journal(&input, &journal)?; + write_journal(&input, &mut journal)?; let records = journal .records .values() @@ -389,7 +402,7 @@ pub fn acknowledge( record.acknowledged = true; } } - write_journal(&input, &journal) + write_journal(&input, &mut journal) }) } @@ -540,11 +553,15 @@ fn read_journal(input: &Path, config: &SourceConfig) -> Result let path = input.join(STATE_FILE); plain_file(&path)?; let config_hash = hash(&serde_json::to_vec(config).map_err(io_error)?); - if !path.exists() { - return Ok(Journal { config_hash, ..Default::default() }); - } - let bytes = read_bounded(&path, MAX_JSON)?; - let journal: Journal = serde_json::from_slice(&bytes).map_err(|_| "source journal is malformed; originals retained".to_owned())?; + let journal: Journal = if let Some(pending) = crate::hot_folder::journal::pending(&path)? { + pending + } else { + if !path.exists() { + return Ok(Journal { config_hash, ..Default::default() }); + } + let bytes = read_bounded(&path, MAX_JSON)?; + serde_json::from_slice(&bytes).map_err(|_| "source journal is malformed; originals retained".to_owned())? + }; if journal.config_hash != config_hash { return Err("source settings changed; choose a fresh staging inbox to avoid applying receipts to another source".into()); } @@ -552,25 +569,64 @@ fn read_journal(input: &Path, config: &SourceConfig) -> Result return Err("source journal exceeds 10000 records".into()); } for (id, record) in &journal.records { - if id.len() != 64 - || !id.bytes().all(|byte| byte.is_ascii_hexdigit()) - || id != &record.source.source_id - || id != &hash(format!("{}\n{}\n{}\n{}", journal.config_hash, record.remote.key, record.remote.revision, record.remote.size).as_bytes()) - || !relative_ok(&record.source.staged_relative) - || record.source.staged_relative.components().count() != 1 - || record.source.sha256.len() != 64 - || !record.source.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) - || !relative_ok(&record.source.relative) - || record.source.relative != record.remote.relative - || record.remote.size == 0 - || record.remote.size > MAX_FILE - { - return Err("source journal contains an invalid receipt path or hash".into()); - } + validate_record(id, record, &journal.config_hash)?; } Ok(journal) } -fn write_journal(input: &Path, journal: &Journal) -> Result<(), String> { +fn validate_record(id: &str, record: &Record, config_hash: &str) -> Result<(), String> { + if id.len() != 64 + || !id.bytes().all(|byte| byte.is_ascii_hexdigit()) + || id != record.source.source_id + || id != hash(format!("{}\n{}\n{}\n{}", config_hash, record.remote.key, record.remote.revision, record.remote.size).as_bytes()) + || !relative_ok(&record.source.staged_relative) + || record.source.staged_relative.components().count() != 1 + || record.source.sha256.len() != 64 + || !record.source.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) + || !relative_ok(&record.source.relative) + || record.source.relative != record.remote.relative + || record.remote.size == 0 + || record.remote.size > MAX_FILE + { + return Err("source journal contains an invalid receipt path or hash".into()); + } + Ok(()) +} + +fn archived_record(input: &Path, config_hash: &str, key: &str) -> Result, String> { + let record: Option = crate::hot_folder::journal::read(&input.join(STATE_FILE), "records", key)?; + if let Some(record) = &record { + validate_record(key, record, config_hash)?; + } + Ok(record) +} + +fn archived_relative(input: &Path, config_hash: &str, relative: &Path) -> Result, String> { + let record: Option = + crate::hot_folder::journal::read(&input.join(STATE_FILE), "paths", &format!("{config_hash}:{}", relative.display()))?; + if let Some(record) = &record { + validate_record(&record.source.source_id, record, config_hash)?; + } + Ok(record) +} + +fn write_journal(input: &Path, journal: &mut Journal) -> Result<(), String> { + let completed: Vec<_> = journal.records.iter().filter(|(_, record)| record.acknowledged).map(|(key, _)| key.clone()).collect(); + let count = completed.len().saturating_sub(256); + let target = input.join(STATE_FILE); + if count > 0 { + crate::hot_folder::journal::begin(&target, journal)?; + } + for key in completed.into_iter().take(count) { + let record = journal.records.get(&key).ok_or("source history changed during compaction")?; + crate::hot_folder::journal::write(&input.join(STATE_FILE), "records", &key, record)?; + crate::hot_folder::journal::write( + &input.join(STATE_FILE), + "paths", + &format!("{}:{}", journal.config_hash, record.remote.relative.display()), + record, + )?; + journal.records.remove(&key); + } let target = input.join(STATE_FILE); plain_file(&target)?; let bytes = serde_json::to_vec(journal).map_err(io_error)?; @@ -580,7 +636,8 @@ fn write_journal(input: &Path, journal: &Journal) -> Result<(), String> { let mut temporary = tempfile::Builder::new().prefix(".pdfcraft-source-state-").tempfile_in(input).map_err(io_error)?; temporary.write_all(&bytes).and_then(|_| temporary.as_file().sync_all()).map_err(io_error)?; temporary.persist(&target).map_err(|error| io_error(error.error))?; - sync_directory(input) + sync_directory(input)?; + crate::hot_folder::journal::finish(&target) } /// A synced temporary file does not make its renamed directory entry durable on Unix. @@ -770,6 +827,55 @@ mod tests { assert_eq!(second.pending_acknowledgements.len(), 128); } #[test] + fn completed_remote_history_rolls_over_and_still_deduplicates_without_downloading() { + let dir = tempfile::tempdir().unwrap(); + let config = SourceConfig::Local; + let mut journal = read_journal(dir.path(), &config).unwrap(); + let mut fake = Fake { count: 300, fail: false }; + let control = Control::new(&|| false); + for _ in 0..3 { + let report = stage_with(&mut fake, &config, dir.path(), &mut journal, &control).unwrap(); + assert!(!report.items.is_empty()); + for item in report.items { + let record = journal.records.get_mut(&item.source_id).unwrap(); + record.ready = true; + record.acknowledged = true; + fs::remove_file(dir.path().join(&item.staged_relative)).unwrap(); + } + write_journal(dir.path(), &mut journal).unwrap(); + } + assert_eq!(journal.records.len(), 256); + let mut restarted = read_journal(dir.path(), &config).unwrap(); + fake.fail = true; // Any attempted download now fails this assertion. + assert!(stage_with(&mut fake, &config, dir.path(), &mut restarted, &control).unwrap().items.is_empty()); + } + + #[test] + fn archived_project_receipts_allow_deleted_originals_but_reject_modified_binding() { + let (root, config, inbox, _) = fixture(SourceAction::Delete); + let store = CredentialStore::OsVault; + let report = stage(&config, &inbox, Some(root.path()), &|| false, &store).unwrap(); + let item = &report.items[0]; + acknowledge(&config, &inbox, item, &item.sha256, Some(root.path()), &|| false, &store).unwrap(); + let mut journal = read_journal(&inbox, &config).unwrap(); + let record = journal.records.remove(&item.source_id).unwrap(); + crate::hot_folder::journal::write(&inbox.join(STATE_FILE), "records", &item.source_id, &record).unwrap(); + crate::hot_folder::journal::write( + &inbox.join(STATE_FILE), + "paths", + &format!("{}:{}", journal.config_hash, record.remote.relative.display()), + &record, + ) + .unwrap(); + write_journal(&inbox, &mut journal).unwrap(); + assert_eq!(stage(&config, &inbox, Some(root.path()), &|| false, &store).unwrap().items.len(), 1); + acknowledge(&config, &inbox, item, &item.sha256, Some(root.path()), &|| false, &store).unwrap(); + let mut forged = record; + forged.remote.key.push_str("changed"); + crate::hot_folder::journal::write(&inbox.join(STATE_FILE), "records", &item.source_id, &forged).unwrap(); + assert!(acknowledge(&config, &inbox, item, &item.sha256, Some(root.path()), &|| false, &store).is_err()); + } + #[test] fn task_json_and_debug_never_contain_credentials() { let config = SourceConfig::Ftp(FtpSource { host: "example.test".into(), diff --git a/crates/engine/src/hot_folder_sources/credentials.rs b/crates/engine/src/hot_folder_sources/credentials.rs index f4df7ab872..673d7bd7f8 100644 --- a/crates/engine/src/hot_folder_sources/credentials.rs +++ b/crates/engine/src/hot_folder_sources/credentials.rs @@ -246,20 +246,39 @@ fn windows_private(path: &Path, directory: bool) -> Result<(), String> { .as_os_str() .security_descriptor(SecurityInformation::Dacl | SecurityInformation::Owner) .map_err(|_| "cannot inspect Windows credential ACL; run provisioning as administrator")?; - windows_descriptor_private(&descriptor, directory) + let account = windows_service_account_sid(); + windows_descriptor_private(&descriptor, directory, account.as_deref()) } #[cfg(windows)] -fn windows_descriptor_private(descriptor: &windows_permissions::SecurityDescriptor, directory: bool) -> Result<(), String> { +fn windows_service_account_sid() -> Option> { + use windows_service::{service::ServiceAccess, service_manager::*}; + // Trust only SCM's administrator-controlled Log On identity, never a SID in + // task JSON or an arbitrary owner. Domain lookup runs inside the bounded child. + let manager = ServiceManager::local_computer(None::<&str>, ServiceManagerAccess::CONNECT).ok()?; + let account = manager.open_service("PdfCraftHotFolder", ServiceAccess::QUERY_CONFIG).ok()?.query_config().ok()?.account_name?; + let name = account.to_str()?; + let (_, sid, _) = winsafe::LookupAccountName(None, name.strip_prefix(".\\").unwrap_or(name)).ok()?; + winsafe::ConvertSidToStringSid(&sid).ok()?.parse().ok() +} + +#[cfg(windows)] +fn windows_descriptor_private( + descriptor: &windows_permissions::SecurityDescriptor, + directory: bool, + service_account: Option<&windows_permissions::Sid>, +) -> Result<(), String> { use windows_permissions::constants::AceType; use windows_permissions::{LocalBox, Sid}; - let allowed: Vec> = ["S-1-5-18", "S-1-5-32-544", "S-1-5-19"] + // NT SERVICE\PdfCraftHotFolder is stable across service-account changes. + let allowed: Vec> = ["S-1-5-18", "S-1-5-32-544", "S-1-5-19", "S-1-5-80-4013549229-3107762393-4025766903-2826249871-1347187068"] .iter() .map(|sid| sid.parse()) .collect::>() .map_err(|_| "cannot construct service ACL identities")?; - if descriptor.owner().is_none_or(|owner| !allowed.iter().any(|sid| owner == &**sid)) { - return Err("credential owner must be SYSTEM, Administrators or LocalService".into()); + let trusted = |owner: &Sid| allowed.iter().any(|sid| owner == &**sid) || service_account == Some(owner); + if descriptor.owner().is_none_or(|owner| !trusted(owner)) { + return Err("credential owner must be SYSTEM, Administrators or the registered service identity".into()); } let acl = descriptor.dacl().ok_or("credential ACL must not grant everyone access")?; if acl.len() == 0 || acl.len() > 32 { @@ -267,8 +286,11 @@ fn windows_descriptor_private(descriptor: &windows_permissions::SecurityDescript } for index in 0..acl.len() { let ace = acl.get_ace(index).ok_or("credential ACL is malformed")?; - if ace.ace_type() != AceType::ACCESS_ALLOWED_ACE_TYPE || ace.sid().is_none_or(|owner| !allowed.iter().any(|sid| owner == &**sid)) { - return Err("credential ACL may grant access only to SYSTEM, Administrators and LocalService; ordinary Users access is forbidden".into()); + if ace.ace_type() != AceType::ACCESS_ALLOWED_ACE_TYPE || ace.sid().is_none_or(|owner| !trusted(owner)) { + return Err( + "credential ACL may grant access only to SYSTEM, Administrators and the service identities; ordinary Users access is forbidden" + .into(), + ); } } if directory { @@ -533,7 +555,13 @@ mod tests { fn windows_service_acl_accepts_only_protected_service_identities() { use windows_permissions::{LocalBox, SecurityDescriptor}; let good: LocalBox = "O:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LS)".parse().unwrap(); - assert!(windows_descriptor_private(&good, true).is_ok()); + assert!(windows_descriptor_private(&good, true, None).is_ok()); + let service_sid = "S-1-5-80-4013549229-3107762393-4025766903-2826249871-1347187068"; + let named_sid: windows_permissions::LocalBox = "S-1-5-21-100-200-300-1001".parse().unwrap(); + let named: LocalBox = + format!("O:{named_sid}D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;{service_sid})").parse().unwrap(); + assert!(windows_descriptor_private(&named, false, Some(&named_sid)).is_ok()); + assert!(windows_descriptor_private(&named, false, None).is_err()); for sddl in [ "O:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LS)(A;OICI;GR;;;BU)", "O:BAD:(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LS)", @@ -542,7 +570,7 @@ mod tests { "O:BA", ] { let descriptor: LocalBox = sddl.parse().unwrap(); - assert!(windows_descriptor_private(&descriptor, true).is_err(), "{sddl}"); + assert!(windows_descriptor_private(&descriptor, true, None).is_err(), "{sddl}"); } let temp = tempfile::tempdir().unwrap(); let store = CredentialStore::ServiceDirectory { directory: temp.path().to_owned() }; diff --git a/crates/engine/src/hot_folder_sources/project.rs b/crates/engine/src/hot_folder_sources/project.rs index e66dffb6e4..2558b13138 100644 --- a/crates/engine/src/hot_folder_sources/project.rs +++ b/crates/engine/src/hot_folder_sources/project.rs @@ -9,11 +9,19 @@ pub(super) fn local_paths(source: &ProjectSource, root: Option<&Path>) -> Result /// An unprocessed missing project entry is an error, never a silently truncated /// grouped output. Completed move/delete entries stay in the immutable manifest. -pub(super) fn validate_and_order(source: &ProjectSource, root: Option<&Path>, journal: &Journal, report: &mut StageReport) -> Result<(), String> { +pub(super) fn validate_and_order( + source: &ProjectSource, + input: &Path, + root: Option<&Path>, + journal: &Journal, + report: &mut StageReport, +) -> Result<(), String> { let (base, manifest) = manifest(source, root)?; for relative in &manifest.files { if fs::symlink_metadata(base.join(relative)).is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound) && !journal.records.values().any(|record| record.remote.relative == *relative) + && !archived_relative(input, &journal.config_hash, relative)? + .is_some_and(|record| record.acknowledged && record.remote.relative == *relative) { return Err("project contains a missing, unprocessed source; restore it before processing the project".into()); } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index b8efe600ae..edba712ea5 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -2058,6 +2058,8 @@ pub struct Session { next_id: u64, /// Seconds since the Unix epoch, injected so saves are deterministic in tests. clock: Option i64>, + /// Stable transaction time for replayable unattended exports. + fixed_timestamp: Option, /// Certificates trusted for signing (Acrobat: Trusted Certificates). trust: Arc, /// Preferences ▸ JavaScript ▸ Enable Acrobat JavaScript, inverted (on by default). @@ -2175,6 +2177,15 @@ impl Session { /// Use a fixed clock (tests) instead of the system time. pub fn with_clock(mut self, clock: fn() -> i64) -> Self { self.clock = Some(clock); + self.fixed_timestamp = None; + self + } + + /// Keep newly written PDF metadata identical when a Hot Folder transaction is replayed. + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn with_timestamp(mut self, timestamp: i64) -> Self { + self.fixed_timestamp = Some(timestamp); + self.clock = None; self } @@ -2184,6 +2195,9 @@ impl Session { } fn now(&self) -> Option { + if let Some(timestamp) = self.fixed_timestamp { + return Some(timestamp); + } if let Some(c) = self.clock { return Some(c()); } diff --git a/crates/hot-folder-service/examples/service_probe.rs b/crates/hot-folder-service/examples/service_probe.rs index 2b1b759e4b..0f4c38a6d0 100644 --- a/crates/hot-folder-service/examples/service_probe.rs +++ b/crates/hot-folder-service/examples/service_probe.rs @@ -7,6 +7,10 @@ use std::{ fn worker(config: &Path, stopping: &AtomicBool) -> Result<(), String> { while !stopping.load(Ordering::SeqCst) { + if config.with_extension("fail").exists() { + std::fs::remove_file(config.with_extension("fail")).map_err(|e| e.to_string())?; + return Err("Synthetic controlled service failure".into()); + } std::fs::write(config.with_extension("heartbeat"), b"running").map_err(|e| e.to_string())?; std::thread::sleep(Duration::from_millis(50)); } @@ -66,6 +70,7 @@ fn main() -> Result<(), String> { "stop" => service::stop(&executable, &config)?, "enable" => service::set_enabled(true, &executable, &config)?, "disable" => service::set_enabled(false, &executable, &config)?, + "prepare-account" => service::prepare_account(&executable, &config)?, "status" => {} _ => return Err("unknown action".into()), } diff --git a/crates/hot-folder-service/src/lib.rs b/crates/hot-folder-service/src/lib.rs index 01d6888bf6..ddceb99418 100644 --- a/crates/hot-folder-service/src/lib.rs +++ b/crates/hot-folder-service/src/lib.rs @@ -94,6 +94,18 @@ pub fn uninstall(executable: &Path, config: &Path) -> Result<(), String> { Ok(()) } +/// Prepare the installed Windows service for an administrator-managed logon account. +/// Does not change credentials, start the service, or grant access to watched folders. +pub fn prepare_account(executable: &Path, config: &Path) -> Result<(), String> { + if !status(executable, config)?.installed { + return Err("Install the optional Hot Folder system service first".into()); + } + #[cfg(windows)] + return windows::prepare_account(); + #[cfg(not(windows))] + Err("prepare-account is a Windows service administration action".into()) +} + /// Blocking service host. The worker must check the cancellation flag between pages. pub fn run(config: &Path, worker: Worker) -> Result<(), String> { path_text(config)?; diff --git a/crates/hot-folder-service/src/windows.rs b/crates/hot-folder-service/src/windows.rs index 4d11882d0c..5ea88329d5 100644 --- a/crates/hot-folder-service/src/windows.rs +++ b/crates/hot-folder-service/src/windows.rs @@ -78,6 +78,25 @@ pub fn uninstall() -> Result<(), String> { Err("Remove the optional Hot Folder component with the PdfCraft installer, which also removes its Windows service.".into()) } +pub fn prepare_account() -> Result<(), String> { + let handle = service(ServiceAccess::CHANGE_CONFIG | ServiceAccess::START) + .map_err(|e| format!("Administrator permission is required to prepare the Windows service: {e}"))?; + // A stable service SID keeps installer-protected data accessible when an + // administrator changes Log On to a domain account or gMSA in Windows SCM. + handle.set_config_service_sid_info(ServiceSidType::Unrestricted).map_err(|e| e.to_string())?; + handle + .update_failure_actions(ServiceFailureActions { + reset_period: ServiceFailureResetPeriod::After(Duration::from_secs(86400)), + reboot_msg: Some("".into()), + command: Some("".into()), + // SCM repeats the final action for subsequent failures. A delay avoids + // a tight crash loop while retaining recovery after prolonged outages. + actions: Some(vec![ServiceAction { action_type: ServiceActionType::Restart, delay: Duration::from_secs(60) }]), + }) + .map_err(|e| e.to_string())?; + handle.set_failure_actions_on_non_crash_failures(true).map_err(|e| e.to_string()) +} + static JOB: OnceLock<(PathBuf, Worker)> = OnceLock::new(); static OUTCOME: OnceLock> = OnceLock::new(); windows_service::define_windows_service!(service_entry, service_main); @@ -124,8 +143,8 @@ fn host() -> Result<(), String> { handle.set_service_status(status_value(ServiceState::Running, false)).map_err(|e| e.to_string())?; let finished = AtomicBool::new(false); let result = std::thread::scope(|scope| { - // Report cancellation while the current page finishes. Bounded wait hints prevent - // Windows from treating a long OCR page as an unresponsive stop request. + // The supervisor terminates its task child within a bounded grace period. + // Keep SCM informed while that child releases its journal lock. let monitor = std::thread::Builder::new() .name("service-stop-status".into()) .spawn_scoped(scope, || { @@ -135,7 +154,7 @@ fn host() -> Result<(), String> { checkpoint = checkpoint.saturating_add(1); let mut state = status_value(ServiceState::StopPending, false); state.checkpoint = checkpoint; - state.wait_hint = Duration::from_secs(120); + state.wait_hint = Duration::from_secs(15); let _ = handle.set_service_status(state); } std::thread::sleep(Duration::from_millis(250)); diff --git a/crates/ui-egui/tests/updates.rs b/crates/ui-egui/tests/release_checks.rs similarity index 100% rename from crates/ui-egui/tests/updates.rs rename to crates/ui-egui/tests/release_checks.rs diff --git a/docs/hot-folder-installation.md b/docs/hot-folder-installation.md index 1daaa838bc..24ffa621de 100644 --- a/docs/hot-folder-installation.md +++ b/docs/hot-folder-installation.md @@ -24,15 +24,75 @@ service startup requires administrator permission. The service runs as `NT AUTHORITY\LocalService`; grant that account access to the folders selected for its tasks. Do not put watched files in the protected program or service-configuration directories. -The MSI registers automatic startup and bounded restart-on-failure recovery. Its task directory -allows administrators and LocalService to write; ordinary users may read status. The separate -`secrets` directory is accessible only to administrators, SYSTEM and LocalService. Credentials +The MSI registers automatic startup and recovery after crashes or controlled service errors, +with a 60-second restart delay (including subsequent failures). An administrator stop does not +trigger recovery. Its task directory allows administrators, LocalService and the service SID +to write; ordinary users may read status. The separate `secrets` directory grants access only +to administrators, SYSTEM, LocalService and `NT SERVICE\PdfCraftHotFolder`. Credentials must never be written into exported task JSON. Installer removal stops and unregisters the service and removes its executable, but retains nonempty task and credential directories. The portable ZIP includes the companion executable but does not register a service. Use the MSI when system boot startup is required. +### Windows Server and authenticated network shares + +Windows Server 2019 (build 17763), 2022 (20348) and 2025 (26100), x64, are target platforms. +A target is not a claim that every server version/NAS has passed runtime acceptance. The +packaged server suite exercises 2022/2025 hosted runners; 2019 requires the explicitly enabled +disposable self-hosted runner described in `packaging/server-tests/README.txt`. Domain/gMSA +integration and the actual NAS remain deployment acceptance checks. + +Use UNC paths such as `\\fileserver\scans\in`, not a drive letter mapped in an interactive +session. Keep `tasks.json`, its limits and all journal/receipt sidecars on the protected **local** +service disk. Input, output and archive paths can be on an SMB share. + +LocalService presents anonymous credentials to remote servers; it is the default for local +folders, not an authenticated NAS identity. An administrator can stop the service, configure +**Log On** in Windows Services for a dedicated domain account (or an already provisioned gMSA), +grant that account share **and** filesystem permissions, then restart it. Grant read access to +inputs and write access to outputs/archives; source move/delete additionally requires write +access to inputs. Windows manages the logon secret; do not put passwords in tasks, scripts or +MSI properties. A gMSA requires the usual AD/host provisioning and service-logon rights. + +The installer prepares a stable service SID, so changing the logon identity does not require +making the private configuration or secret store readable by other users. The following +explicit elevated command reapplies service-SID/recovery settings to a matching installation: + +```powershell +& 'C:\Program Files\PdfCraft\pdfcraft-hot-folder.exe' --service-action prepare-account ` + --config 'C:\ProgramData\PdfCraft\HotFolder\tasks.json' +``` + +It does not change the account, password or share permissions. Stop/start after changing the +identity. MSI upgrades preserve the registered service/account using stable component GUIDs +and replace packaged binaries even when preview file versions match; user task/secret files +are never packaged payload. Do not change those GUIDs when modifying this installer. + +### Bounded task workers + +The system service supervises one isolated task process at a time. Unavailable shares, blocked +I/O, OCR hangs and task crashes cannot hold the supervisor indefinitely. Each task pass defaults +to 300 seconds and a 4096 MiB memory budget. Windows uses a hard Job Object commit-memory +limit; Unix uses a sampled resident-memory limit. Large documents or grouped batches may need a larger +explicit deadline. Set the optional local `tasks.worker.json` beside `tasks.json`, then restart: + +```json +{ "task_timeout_seconds": 1800, "memory_limit_mib": 4096 } +``` + +Allowed values are 10–86400 seconds and 256–65536 MiB. A stop closes the child control pipe; +after a 3-second grace period the supervisor terminates the child and waits at most 5 further +seconds. Windows job objects also terminate children when the supervisor dies. A task failure +is recorded with backoff; healthy tasks continue. Diagnostic-log write failure is reported to +stderr without stopping otherwise valid scheduling. Corrupt receipt data is never reset to +empty state. These boundaries do not promise recovery from a failing OS/kernel or storage that +cannot persist a journal; such failures are surfaced for administration. + +References: [LocalService network identity](https://learn.microsoft.com/en-us/windows/win32/services/localservice-account), +[managed service accounts](https://learn.microsoft.com/en-us/windows-server/identity/ad-ds/manage/understand-service-accounts), +[SCM failure recovery](https://learn.microsoft.com/en-us/windows/win32/api/winsvc/ns-winsvc-service_failure_actions_flag). + ## macOS Use the `.pkg` installer and select **PdfCraft Hot Folder** in Customize. The selection is off @@ -103,3 +163,11 @@ start/stop, startup enable/disable and graceful cancellation. Installer tests co MSI/PKG/DEB definitions around the synthetic worker and verify optional installation, task preservation and removal. These run only on disposable CI machines. They do not replace testing the final OCR executable and signed installer artifacts on each supported architecture. + +`.github/workflows/hot-folder-server.yml` is the separate, manual acceptance suite for an +already-successful matching installer build. It installs the real x64 OCR service on disposable +Windows Server 2022/2025 hosts, exercises local folders and authenticated SMB, output outages, +worker crashes, bounded cancellation and same-version account-preserving upgrade. The optional +Server 2019 path requires an explicitly configured disposable self-hosted runner; hosted CI +cannot establish Server 2019 compatibility. See `packaging/server-tests/README.txt` for the +provenance checks and invocation. A checked-in suite is not evidence of a successful run. diff --git a/docs/hot-folder.md b/docs/hot-folder.md index b3ad2c548c..c0c45babe2 100644 --- a/docs/hot-folder.md +++ b/docs/hot-folder.md @@ -1,6 +1,6 @@ # PdfCraft Hot Folder -**Status: implemented native workflow, in final integration review. Proprietary ABBYY task/project formats are not compatible.** +**Status: implemented native workflow, with isolated task workers and durable restart recovery. Platform package acceptance is documented separately. Proprietary ABBYY task/project formats are not compatible.** The optional standalone application processes folders without keeping the PdfCraft editor open. It shares OCR, orientation correction, deskew and paper-edge detection with the engine. Its @@ -41,8 +41,11 @@ independently from the system language, using the shared OCR language list. - Analyze-only mode writes versioned JSON reports with existing text, geometry, image rectangles and metadata. It bypasses OCR, scan corrections and blank-page removal, and supports the selected page range. An analysis saving step after normal processing describes the resulting PDF; the analyzer itself never runs OCR. - Optional blank-page archives preserve removed pages as separate PDFs; originals are not removed until these files also save successfully. An entirely blank input requires an archive destination. -- Durable receipts suppress duplicate processing across worker restarts. Failed files are logged - once, retried when their content changes or through Retry/Start. Stop is checked between pages +- Durable receipts suppress duplicate processing across worker restarts. Transient filesystem, + publication and task-process failures retry automatically with bounded exponential backoff + (5 seconds up to 15 minutes); one offline task does not block the others. A malformed unchanged + input is quarantined until its contents change or Retry/Start is requested. A permanently failed + group stays blocked rather than silently omitting one member. Stop is checked between pages and does not save partial OCR output. Editing a running task through automation or its configuration file also cancels the previous settings snapshot, including a change from Delete to Keep. - Import/export task JSON, duplicate tasks, inspect the last 1,000 log events, filter failures and @@ -60,6 +63,7 @@ settling time or have the producer rename the completed file into the input fold `pdfcraft-hot-folder --once --config tasks.json` executes one scheduler pass and prints JSON. `pdfcraft-hot-folder --service --config tasks.json` hosts the installed system service. `--service-action status|start|stop|enable|disable|install|uninstall` explicitly administers it. +Windows `prepare-account` reapplies service SID/recovery policy without changing the SCM account. Unix installation additionally requires `--account USER`, a non-root service account. Unix processing refuses root; elevated execution is reserved for setup and administration. `--initialize-config --config PATH` atomically creates a stopped configuration without replacing @@ -82,7 +86,8 @@ variable, Outlook device sign-in and folder listing. Secrets never appear in too ## Remote credentials and mail handling The service uses a separate protected `secrets` directory next to its configuration. Windows -installation restricts this to SYSTEM, Administrators and LocalService. Unix requires the service +installation restricts this to SYSTEM, Administrators, LocalService and the fixed service SID. +A separately configured SCM service account is validated by its actual OS SID, not a task-JSON name. Unix requires the service account as owner and permissions 0700 for the directory and 0600 for files. Unsafe permissions, symlinks and missing provisioning are rejected instead of falling back to an unprotected file. Set up credentials under the service identity, or use an administrator task-manager session @@ -120,10 +125,26 @@ Example task (folders must already exist): } ``` -`tasks.state.json` stores receipts, queue observations and logs. `tasks.lock` and -`tasks.worker.lock` are OS locks, released automatically after a crash. Corrupt JSON is reported -without resetting or overwriting it. Do not delete the journal unless you intentionally want -to process inputs again. Keep it together with its configuration when archiving a completed job. +`tasks.state.json` keeps the active receipts/observations bounded to 256 entries and 2 MiB +per map; logs keep at most 1,000 events and 1 MiB. Exact historical records spill to sharded +files under `tasks.journal`, so keeping more than 10,000 originals does not stop the queue. +Disk history grows with each distinct source version; provision disk space accordingly. +`tasks.state.pending.json` is the atomic write-ahead snapshot: recovery exposes all grouped +receipts together even if the process died during history spill. Output names and PDF +metadata use the persisted attempt timestamp, making replayed OCR/export byte-identical. + +`tasks.lock` and `tasks.worker.lock` are OS locks, released automatically after a crash. +Corrupt JSON is reported without resetting or overwriting it. Back up the configuration, +state, pending state and complete journal together. Do not remove individual history files +to free space: doing so discards deduplication evidence. Configuration, journals and worker +limits belong on local storage; input/output folders may be network shares. + +The parent service delegates each due task to a bounded child process. The default is +300 seconds and 4096 MiB per child; see [installation and limits](hot-folder-installation.md). +Windows enforces a Job Object commit-memory limit and kills children if the parent dies. +Unix samples resident memory; it does not promise an equivalent kernel commit limit. +An idle pass does not rewrite unchanged state. Updating one manifest cannot newly overlap +another active task's source or output paths. ## Functional reference and remaining scope diff --git a/docs/roadmap.md b/docs/roadmap.md index 977f725492..f21f940c84 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -77,7 +77,7 @@ total matches the parity estimate. | M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
    Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; Create PDF from multiple files (PDFs, images and text, edited as pages in the grid). Done since: German/French/Spanish/Italian/Dutch/Danish/Norwegian/Swedish/Finnish/Portuguese/horizontal-Japanese OCR and system-language UI defaults, Unicode text layers, automatic quarter-turn correction, bounded deskew, and automatic paper-edge crop with safety padding through UI and automation. Missing: additional OCR languages, deskew with annotations, editable-text OCR output, Office export/import, web printing | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools).
    Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export. Missing: autotag, Tags/Order/Content panels, Reading Order tool, alt-text workflow, compare, geospatial/3D measurement, search index, XFA | -| M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. In final integration review: separate Hot Folder application, durable local/FTP/Outlook/project processing, calendar scheduling, grouped export/analysis, restricted boot services and agent tools; scope in [hot-folder.md](hot-folder.md). Missing: Action Wizard, AI providers | +| M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Implemented Hot Folder application, durable local/FTP/Outlook/project processing, calendar scheduling, grouped export/analysis, restricted boot services and agent tools. Server hardening adds isolated bounded workers, recoverable journals, transient retries and custom service-account upgrades; actual package/server acceptance is separate; scope in [hot-folder.md](hot-folder.md). Missing: Action Wizard, AI providers | | M14 | 1.0 polish: performance, localization, installers | 120–250 | 15% | 80–150 | Done: PhotoCraft's translation system (`i18n/`: TSV catalogs, `tl!`, command-id and plural entries, system-language detection, strict catalog tests); every dialog, panel and notice goes through `tl!`. Catalogs of ≈ 2,000–2,140 entries for Japanese, Simplified and Traditional Chinese, Russian, Bulgarian, German, Spanish, French, Telugu, Hungarian, Ukrainian, Italian, Brazilian Portuguese and Arabic (logical-order catalog put into display order; layout not yet mirrored); Czech covers the menus. Signed and notarized macOS builds, signed Windows installers (x64, x86, ARM64), AppImage, deb, rpm, Flatpak and FreeBSD packages. #307 (bounded large-PDF memory): shared resource indexes and parser metadata, lazy inspection, bounded caches; a 9,156-page file inspects in ≈ 1.8 s at ≈ 500 MiB instead of exceeding 10 GiB. Shortcut display uses localized modifier/key names across menus, toolbar, palette, Help and tooltips (#496). Missing: a right-to-left layout, a Chinese UI face in releases, Hindi/Indonesian/Korean/Vietnamese, file-backed input and global memory budgets, performance budgets, keyboard-only operation | | | **Total** | **2,085–3,840** | **≈ 49% ready** | **≈ 770–1,400** | Area view of the same work: [target-app-parity.md](target-app-parity.md#by-feature-area) | diff --git a/packaging/server-tests/README.txt b/packaging/server-tests/README.txt new file mode 100644 index 0000000000..5292f814f5 --- /dev/null +++ b/packaging/server-tests/README.txt @@ -0,0 +1,111 @@ +Windows Server acceptance of the packaged Hot Folder OCR service +================================================================ + +Run only on an elevated, disposable GitHub Actions Windows Server runner. +acceptance.ps1 installs the opt-in MSI, changes a local service logon right, +creates an ephemeral non-admin account and an encrypted SMB share, and injects +process/filesystem faults. Never run it on a workstation or production server. +It refuses an existing PdfCraft installation, service or task directory. + +The suite uses the actual packaged CLI, companion and OCR models. All PDFs and +images are generated locally from original synthetic text with PdfCraft's +embedded open fonts. No private scans, external corpus or system fonts are used. +No service account password is accepted from a user: CI generates one, masks it, +and supplies it directly to SCM through CIM, never through command-line arguments, +task JSON or logs. The account, share, service and prior service-logon-right +assignment are cleaned up independently, including after an earlier cleanup error. +Only bounded result/log files are uploaded; the private working directory is not. +Cancellation or host loss can bypass finally blocks: self-hosted hosts must be +ephemeral and destroyed after every run, even when the workflow is cancelled. + +How to run +---------- +1. Review and push the exact source commit, including this suite. +2. Finish a successful preview-installers.yml or release.yml workflow at that SHA. + Its event must be push, workflow_dispatch or release, in this repository. +3. Dispatch hot-folder-server.yml on the same ref with installer_run_id, + expected_sha (full 40 characters), and artifact_name. The default artifact name + is pdfcraft-preview-windows-x64. The MSI SHA256 file must accompany the MSI. +4. Read both hot-folder-server-windows-2022 and hot-folder-server-windows-2025 + artifacts. A passing report.json and successful cleanup are required. + +Example (documentation only; replace all placeholders): + gh workflow run hot-folder-server.yml --ref REVIEWED_REF \ + -f installer_run_id=SUCCESSFUL_RUN_ID -f expected_sha=FULL_COMMIT_SHA + +Before this new workflow reaches the default branch, an existing installer +workflow can invoke it as a reusable workflow in a SECOND manual dispatch, passing +the previous successful installer run ID and skipping its own build jobs. Calling +it from the same still-running installer run is intentionally rejected. All jobs +and the script require the original event to be workflow_dispatch; workflow_call +does not provide a bypass for pull requests, pushes or other automatic callers. + +The preview workflow provides that second mode explicitly: + gh workflow run preview-installers.yml --ref REVIEWED_REF \ + -f mode=server-acceptance -f installer_run_id=SUCCESSFUL_RUN_ID +It skips all installer/processing builds, passes its own exact github.sha as the +expected_sha, and invokes the local reusable workflow from that same commit. +Do not advance REVIEWED_REF between the installer run and acceptance dispatch. +The default mode remains build; pushes never launch the acceptance suite. + +Windows Server 2019 +------------------- +GitHub retired its hosted Windows Server 2019 image on 30 June 2025. The optional +manual job requires a dedicated, elevated, disposable x64 Server 2019 runner with +labels self-hosted, Windows, X64, pdfcraft-server-2019-disposable. Configure the +hot-folder-server-2019 environment with REQUIRED REVIEWERS before enabling it; +declaring an environment in YAML alone does not enforce review. Keep this runner +out of general runner groups and never run unreviewed PR code on it. Explicitly set +include_server_2019=true and server_2019_confirmation=DISPOSABLE SERVER 2019. +Until that real run passes, Server 2019 is prepared for testing, not runtime-proven. + +What a passing run proves +------------------------ +- Fresh opt-in MSI registration defaults to LocalService, starts automatically, + creates no enabled tasks, and protects data/secrets with the stable service SID. +- Actual OCR creates searchable PDF and TXT from an image-only PDF; original bytes + are preserved. The PDF is reopened and searched through the packaged CLI. +- A write-denied output retains its input until both outputs succeed. Another local + task continues, and the failed task recovers automatically without manual retry. +- A malformed single PDF is retained while a later valid document is processed. +- Killing the real service during a CPU-active OCR child terminates that child; + SCM restarts the service and the interrupted work completes without duplicate + outputs. Stopping during OCR is bounded. A controlled initialization Err also + recovers through SCM, whose configured delay is 60 seconds. +- An ephemeral non-admin service account authenticates to an encryption-required + loopback SMB share. Removing that share isolates the failure to its task; a local + task continues and the UNC task recovers automatically after share recreation. +- A same-version major upgrade preserves task bytes, all executable bytes and the + SCM account/password, proven by subsequent authenticated SMB OCR without supplying + the password again. Its test-only MSI copy changes ProductCode and PackageCode + GUIDs but not embedded binaries. This modifies any original signature; it is never + published or delivered as an installer. A separate probe installer test checks + replacement with changed binary payload, including equal-version replacement. + +Limits: this is authenticated loopback SMB, not certification of every NAS vendor, +domain controller, gMSA deployment or WAN failure mode. It does not perform a full +server reboot. It verifies automatic startup configuration and real SCM restart, +not a reboot result. Server versions are reported only after their actual runs. + +Local validation without installation +------------------------------------ + pwsh -NoProfile -File packaging/server-tests/check.ps1 + actionlint -config-file packaging/server-tests/actionlint.yaml \ + .github/workflows/hot-folder-server.yml +check.ps1 only parses scripts, exercises confinement and mocks the GitHub metadata +verifier; it never runs acceptance.ps1 or accesses a real service/account/share. + +Primary references +------------------ +Hosted images and available labels: + https://github.com/actions/runner-images +Windows Server 2019 hosted runner retirement: + https://github.com/actions/runner-images/issues/12045 +LocalService presents anonymous network credentials (hence the separate SMB phase): + https://learn.microsoft.com/en-us/windows/win32/services/localservice-account +SCM account/password configuration using Win32_Service.Change: + https://learn.microsoft.com/en-us/windows/win32/cimwin32prov/change-method-in-class-win32-service +SMB encryption-required share: + https://learn.microsoft.com/en-us/powershell/module/smbshare/new-smbshare +Reusable workflow event context: + https://docs.github.com/en/actions/reference/workflows-and-actions/reusing-workflow-configurations diff --git a/packaging/server-tests/acceptance.ps1 b/packaging/server-tests/acceptance.ps1 new file mode 100644 index 0000000000..dc939b781d --- /dev/null +++ b/packaging/server-tests/acceptance.ps1 @@ -0,0 +1,358 @@ +<# +Acceptance of an actual packaged OCR service on a DISPOSABLE elevated CI server. +Installs/uninstalls the MSI, creates a temporary non-admin user and encrypted SMB +share, and deliberately kills only the installed test service. Never run on a +personal/work server. Guards require GitHub Actions, explicit consent, supported +Server build, no existing PdfCraft installation/configuration, and admin identity. +#> +param( + [Parameter(Mandatory)][string] $InstallerDirectory, + [Parameter(Mandatory)][ValidatePattern('^[a-f0-9]{40}$')][string] $ExpectedCommit, + [int] $ExpectedServerBuild = 0, + [switch] $DisposableHost +) +. "$PSScriptRoot/common.ps1" + +Assert-That ($DisposableHost -and $env:GITHUB_ACTIONS -eq 'true') 'This installs a service: explicit disposable GitHub Actions host required.' +Assert-That ($env:GITHUB_EVENT_NAME -eq 'workflow_dispatch') 'The server acceptance suite is manual only.' +Assert-That ($ExpectedCommit -ceq $env:GITHUB_SHA) 'Checked-out workflow and installer commit must match.' +$identity = [Security.Principal.WindowsIdentity]::GetCurrent() +$principal = [Security.Principal.WindowsPrincipal]::new($identity) +Assert-That ($principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) 'An elevated disposable runner is required.' +$os = Get-CimInstance Win32_OperatingSystem +Assert-That ($os.ProductType -ne 1 -and [int] $os.BuildNumber -in @(17763, 20348, 26100)) 'Expected Windows Server 2019, 2022 or 2025.' +if ($ExpectedServerBuild) { Assert-That ([int] $os.BuildNumber -eq $ExpectedServerBuild) 'Self-hosted runner is not the explicitly selected Server version.' } +Assert-That ([Runtime.InteropServices.RuntimeInformation]::OSArchitecture -eq 'X64') 'This acceptance suite consumes the native x64 installer.' +Assert-That (-not (Get-HotService)) 'Refusing to change an existing PdfCraftHotFolder service.' +$script:DataDirectory = Join-Path $env:ProgramData 'PdfCraft/HotFolder' +Assert-That (-not (Test-Path -LiteralPath $script:DataDirectory)) 'Refusing to use an existing Hot Folder configuration directory.' +Assert-That (-not (Test-Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\pdfcraft.exe')) 'Refusing to change an existing PdfCraft installation.' + +$root = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) +$script:Results = Join-Path $root 'server-test-results' +Assert-That (-not (Test-Path -LiteralPath $script:Results)) 'Acceptance results already exist; use a fresh checkout.' +[void] (New-Item -ItemType Directory -Path $script:Results) +$token = [guid]::NewGuid().ToString('N') +$script:TestRoot = Join-Path $env:RUNNER_TEMP "pdfcraft-server-$token" +[void] (Assert-SafePath $script:TestRoot $env:RUNNER_TEMP) +[void] (New-Item -ItemType Directory -Path $script:TestRoot) +Set-PrivateDirectory $script:TestRoot @('S-1-5-19') +$appDirectory = Join-Path $script:TestRoot 'app' +$script:Cli = Join-Path $appDirectory 'pdfcraft-cli.exe' +$companion = Join-Path $appDirectory 'pdfcraft-hot-folder.exe' +$script:Config = Join-Path $script:DataDirectory 'tasks.json' +$marker = Join-Path $script:DataDirectory 'CI-OWNER.txt' +$msiExe = Join-Path $env:SystemRoot 'System32/msiexec.exe' +$scExe = Join-Path $env:SystemRoot 'System32/sc.exe' +$secedit = Join-Path $env:SystemRoot 'System32/secedit.exe' +$user = 'pfc' + $token.Substring(0, 12) +$account = "$env:COMPUTERNAME\$user" +$shareName = 'PdfCraftCI-' + $token.Substring(0, 12) + '$' +$shareRoot = Join-Path $script:TestRoot 'smb' +$sharePath = "\\$env:COMPUTERNAME\$shareName" +$password = $null +$accountCreated = $false +$privilegeChanged = $false +$installed = $false +$productCodes = [Collections.Generic.List[string]]::new() +$evidence = [ordered]@{ + result = 'running'; os = $os.Caption; build = $os.BuildNumber + source_commit = $ExpectedCommit; installer_run = $env:INSTALLER_RUN_ID + tests = [ordered]@{}; limitation = 'Authenticated encrypted loopback SMB, not a specific NAS vendor or domain/gMSA integration.' +} + +function Record([string] $Name, $Value) { + $script:evidence.tests[$Name] = $Value + Write-Json (Join-Path $script:Results 'report.json') $script:evidence + Write-Output "passed: $Name" +} + +function New-TestFolder([string] $Name) { + $path = Join-Path $script:TestRoot $Name + [void] (Assert-SafePath $path $script:TestRoot) + [void] (New-Item -ItemType Directory -Path $path) + return $path +} + +function Put-Scan([string] $Folder, [string] $Stem, [string] $Source = $script:Scan) { + Copy-Item -LiteralPath $Source -Destination (Join-Path $Folder "$Stem.pdf") +} + +function Wait-TaskChild([string] $TaskId) { + Wait-For "active OCR child for $TaskId" { + $children = @(Get-CimInstance Win32_Process -Filter "Name='pdfcraft-hot-folder.exe'" | Where-Object { + $_.ExecutablePath -eq $companion -and $_.CommandLine -match '--task-child' -and + $_.CommandLine -match ('--task-id\s+"?' + [regex]::Escape($TaskId) + '(?:"|\s|$)') -and + ([uint64] $_.KernelModeTime + [uint64] $_.UserModeTime) -gt 1000000 + }) + if ($children.Count) { $script:ObservedChild = $children[0].ProcessId; return $true } + return $false + } 80 + return $script:ObservedChild +} + +function New-TestShare { + [void] (New-SmbShare -Name $shareName -Path $shareRoot -FullAccess @($account, $identity.Name) -EncryptData $true -CachingMode None) +} + +function Set-ServiceLogonRight([string] $Assignments, [string] $Name) { + $policy = Join-Path $script:TestRoot "$Name.inf" + $content = "[Unicode]`r`nUnicode=yes`r`n[Version]`r`nsignature=`"`$CHICAGO`$`"`r`nRevision=1`r`n[Privilege Rights]`r`nSeServiceLogonRight = $Assignments`r`n" + [IO.File]::WriteAllText($policy, $content, [Text.Encoding]::Unicode) + [void] (Invoke-Program $secedit @('/configure', '/db', (Join-Path $script:TestRoot "$Name.sdb"), '/cfg', $policy, '/areas', 'USER_RIGHTS', '/quiet') $Name) +} + +try { + $msis = @(Get-ChildItem -LiteralPath $InstallerDirectory -File | Where-Object { $_.Name -match '^pdfcraft-[A-Za-z0-9_.-]+-windows-x64\.msi$' }) + Assert-That ($msis.Count -eq 1) 'Expected exactly one x64 MSI in the verified artifact.' + $msi = $msis[0].FullName + $sums = @(Get-ChildItem -LiteralPath $InstallerDirectory -File | Where-Object { $_.Name -eq ($msis[0].Name + '.sha256') -or $_.Name -eq 'SHA256SUMS.txt' }) + Assert-That ($sums.Count -eq 1) 'Expected exactly one installer checksum file.' + $expectedHash = @((Get-Content -LiteralPath $sums[0].FullName) | Where-Object { $_ -match ('^[a-fA-F0-9]{64}\s+\*?' + [regex]::Escape($msis[0].Name) + '$') }) + Assert-That ($expectedHash.Count -eq 1) 'No unambiguous checksum record for this MSI.' + $digest = (Get-FileHash -LiteralPath $msi -Algorithm SHA256).Hash.ToLowerInvariant() + Assert-That ($digest -eq ($expectedHash[0] -split '\s+')[0].ToLowerInvariant()) 'Installer checksum mismatch.' + $evidence.installer_sha256 = $digest + $productCodes.Add((Get-ProductCode $msi)) + # Include a partially failed installation in the cleanup attempt too; each + # product code came from this run's MSI, and pre-existing PdfCraft is refused. + $installed = $true + [void] (Invoke-Program $msiExe @('/i', $msi, '/qn', '/norestart', 'ALLUSERS=1', 'INSTALLHOTFOLDER=1', 'INSTALLDESKTOPSHORTCUT=0', "INSTALLFOLDER=$appDirectory", '/L*v', (Join-Path $script:Results 'install-msi.log')) 'install' 180 @(0, 3010)) + Wait-For 'empty service-owned configuration' { Test-Path -LiteralPath $script:Config } 60 + [IO.File]::WriteAllText($marker, $token) + $first = Get-Content -LiteralPath $script:Config -Raw | ConvertFrom-Json + Assert-That (@($first.tasks).Count -eq 0) 'Installer unexpectedly enabled processing tasks.' + $service = Get-HotService + Assert-That ($service.StartName -eq 'NT AUTHORITY\LocalService') 'Fresh installation must default to LocalService.' + Assert-That ($service.StartMode -eq 'Auto' -and $service.State -eq 'Running') 'Installed service must be running with automatic startup.' + $serviceSid = ([Security.Principal.NTAccount]::new('NT SERVICE\PdfCraftHotFolder')).Translate([Security.Principal.SecurityIdentifier]).Value + Set-PrivateDirectory $script:TestRoot @('S-1-5-19', $serviceSid) + foreach ($folder in @($script:DataDirectory, (Join-Path $script:DataDirectory 'secrets'))) { + $acl = Get-Acl -LiteralPath $folder + Assert-That $acl.AreAccessRulesProtected 'Installed service directory DACL must be protected.' + $sids = @($acl.Access | ForEach-Object { $_.IdentityReference.Translate([Security.Principal.SecurityIdentifier]).Value }) + Assert-That ($serviceSid -in $sids) 'Stable service-SID ACE missing from the installed directory.' + if ($folder.EndsWith('\secrets')) { Assert-That ('S-1-5-32-545' -notin $sids -and 'S-1-1-0' -notin $sids) 'Credentials directory must not grant Users/Everyone access.' } + } + $registry = Get-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Services\PdfCraftHotFolder' + Assert-That ($registry.ServiceSidType -eq 1 -and $registry.FailureActionsOnNonCrashFailures -eq 1) 'Service SID or non-crash recovery is not enabled.' + foreach ($query in @('qfailure', 'qfailureflag', 'qsidtype')) { [void] (Invoke-Program $scExe @($query, 'PdfCraftHotFolder') $query) } + Record 'installed_localservice_acl_recovery' @{ account = $service.StartName; service_sid = $serviceSid; tasks_initially_empty = $true } + [void] (Stop-HotService) + $env:PDFCRAFT_SYSTEM_FONTS = '0' + $env:PDFCRAFT_MODELS = Join-Path $appDirectory 'models' + $ocr = Invoke-Tool 'ocr-status' 'ocr_status' @{} + Assert-That $ocr.available 'Packaged modern OCR models unavailable.' + $text = ('Hot folder searchable output from a synthetic server acceptance scan.' + "`n") * 12 + $script:Scan = Join-Path $script:TestRoot 'scan.pdf' + $scanImage = Join-Path $script:TestRoot 'scan.png' + $largeScan = Join-Path $script:TestRoot 'large-scan.pdf' + [void] (Invoke-CliScript 'create-scans' @( + @{ tool = 'doc_create'; args = @{ from = 'text'; text = $text } }, + @{ tool = 'page_render'; args = @{ doc = 1; page = 1; dpi = 150 }; out = $scanImage }, + @{ tool = 'doc_create'; args = @{ from = 'images'; paths = @($scanImage); dpi = 150 } }, + @{ tool = 'doc_save'; args = @{ doc = 2; path = $script:Scan } }, + @{ tool = 'doc_create'; args = @{ from = 'images'; paths = @($scanImage) * 24; dpi = 150 } }, + @{ tool = 'doc_save'; args = @{ doc = 3; path = $largeScan } } + )) + $scanHash = (Get-FileHash -LiteralPath $script:Scan -Algorithm SHA256).Hash + $localIn = New-TestFolder 'local-in'; $localOut = New-TestFolder 'local-out' + $faultIn = New-TestFolder 'fault-in'; $faultOut = New-TestFolder 'fault-out' + $localTask = New-Task 'local' $localIn $localOut + $faultTask = New-Task 'fault' $faultIn $faultOut 'delete' + Save-Tasks @($localTask, $faultTask) 'configure-local' + Put-Scan $localIn 'baseline' + Start-HotService + Wait-Output $localOut 'baseline' + $search = Invoke-CliScript 'search-saved' @( + @{ tool = 'doc_open'; args = @{ path = (Join-Path $localOut 'baseline.pdf') } }, + @{ tool = 'text_find'; args = @{ doc = 1; query = 'searchable output' } } + ) + Assert-That ($search -match '"matches"\s*:\s*\[\s*\{') 'Saved service OCR PDF does not contain searchable text.' + Assert-That ((Get-FileHash -LiteralPath (Join-Path $localIn 'baseline.pdf')).Hash -eq $scanHash) 'Local source changed.' + Record 'actual_localservice_ocr' @{ source_unchanged = $true; searchable_pdf = $true; text_export = $true } + + # Write denial is recoverable I/O, and cannot justify deleting the input or + # blocking an independent task. The denial targets only this test service SID. + $deny = [Security.AccessControl.FileSystemAccessRule]::new([Security.Principal.SecurityIdentifier]::new($serviceSid), 'Write', 'ContainerInherit,ObjectInherit', 'None', 'Deny') + $faultAcl = Get-Acl -LiteralPath $faultOut + $faultAcl.AddAccessRule($deny); Set-Acl -LiteralPath $faultOut -AclObject $faultAcl + Put-Scan $faultIn 'write-denied' + Wait-For 'recorded transient output failure' { [bool] (Read-State).tasks.fault.error } 90 + Assert-That ((Get-FileHash -LiteralPath (Join-Path $faultIn 'write-denied.pdf')).Hash -eq $scanHash) 'Failed output discarded or changed source.' + Put-Scan $localIn 'during-output-failure' + Wait-Output $localOut 'during-output-failure' 90 + $faultAcl.RemoveAccessRuleSpecific($deny); Set-Acl -LiteralPath $faultOut -AclObject $faultAcl + Wait-Output $faultOut 'write-denied' 180 + Wait-For 'source deleted only after both outputs' { -not (Test-Path -LiteralPath (Join-Path $faultIn 'write-denied.pdf')) } 30 + Record 'output_failure_isolated_and_auto_retried' @{ independent_task_completed = $true; no_manual_retry = $true; source_kept_until_outputs = $true } + + [IO.File]::WriteAllText((Join-Path $localIn 'broken.pdf'), "%PDF-1.7`nnot a valid PDF", [Text.Encoding]::ASCII) + Put-Scan $localIn 'after-broken-document' + Wait-For 'permanent malformed-document error' { (Read-State).tasks.local.failed -gt 0 } 90 + Wait-Output $localOut 'after-broken-document' 90 + Record 'malformed_document_does_not_block_next_file' @{ malformed_source_retained = (Test-Path -LiteralPath (Join-Path $localIn 'broken.pdf')); valid_next_file_completed = $true } + + Put-Scan $localIn 'crash-recovery' $largeScan + $largeHash = (Get-FileHash -LiteralPath $largeScan).Hash + $childPid = Wait-TaskChild 'local' + $oldPid = (Get-HotService).ProcessId + Assert-That ($oldPid -gt 0 -and (Get-Process -Id $oldPid).Path -eq $companion) 'Refusing to kill an unrelated service process.' + Stop-Process -Id $oldPid -Force + Wait-For 'orphaned OCR child terminated by service job' { -not (Get-Process -Id $childPid -ErrorAction SilentlyContinue) } 15 + Wait-For 'automatic SCM recovery after active OCR hard kill' { $s = Get-HotService; $s.State -eq 'Running' -and $s.ProcessId -ne $oldPid } 150 + Wait-Output $localOut 'crash-recovery' 240 + Assert-That ((Get-FileHash -LiteralPath (Join-Path $localIn 'crash-recovery.pdf')).Hash -eq $largeHash) 'Crash recovery changed its source.' + $hashesBefore = @(Get-ChildItem -LiteralPath $localOut -File | ForEach-Object { "$($_.Name):$((Get-FileHash -LiteralPath $_.FullName).Hash)" }) + Start-Sleep -Seconds 5 + $hashesAfter = @(Get-ChildItem -LiteralPath $localOut -File | ForEach-Object { "$($_.Name):$((Get-FileHash -LiteralPath $_.FullName).Hash)" }) + Assert-That (-not (Compare-Object $hashesBefore $hashesAfter)) 'Recovered queue duplicated or modified completed outputs.' + Assert-That (@(Get-ChildItem -LiteralPath $localOut -File | Where-Object Name -Like 'crash-recovery*').Count -eq 2) 'Crash recovery emitted duplicate PDF/TXT outputs.' + Record 'active_ocr_hardkill_recovery' @{ old_service_pid = $oldPid; active_child_pid = $childPid; recovered_service_pid = (Get-HotService).ProcessId; source_unchanged = $true; outputs_idempotent = $true } + + Put-Scan $localIn 'bounded-stop' $largeScan + $stopChild = Wait-TaskChild 'local' + $stopSeconds = Stop-HotService 35 + Wait-For 'stopped service child gone' { -not (Get-Process -Id $stopChild -ErrorAction SilentlyContinue) } 10 + Assert-That ($stopSeconds -lt 35) 'Service stop exceeded the acceptance bound.' + Record 'bounded_stop_during_ocr' @{ seconds = $stopSeconds; source_retained = (Test-Path -LiteralPath (Join-Path $localIn 'bounded-stop.pdf')) } + + # A filesystem error during worker initialization returns Err rather than a + # crash. Recovery must include non-crash exits, without a manual second start. + $workerLock = [IO.Path]::ChangeExtension($script:Config, 'worker.lock') + $workerBackup = $workerLock + '.ci-backup' + [void] (Assert-SafePath $workerLock $script:DataDirectory) + [void] (Assert-SafePath $workerBackup $script:DataDirectory) + if (Test-Path -LiteralPath $workerLock) { Move-Item -LiteralPath $workerLock -Destination $workerBackup } + [void] (New-Item -ItemType Directory -Path $workerLock) + try { + $request = Invoke-CimMethod -InputObject (Get-HotService) -MethodName StartService + Assert-That ($request.ReturnValue -in @(0, 7, 8, 9)) "Unexpected controlled-start result: $($request.ReturnValue)" + Wait-For 'controlled non-crash service failure' { $s = Get-HotService; $s.State -eq 'Stopped' -and $s.ExitCode -eq 1066 -and $s.ServiceSpecificExitCode -eq 1 } 35 + } finally { + [void] (Assert-SafePath $workerLock $script:DataDirectory) + Remove-Item -LiteralPath $workerLock -Force + if (Test-Path -LiteralPath $workerBackup) { Move-Item -LiteralPath $workerBackup -Destination $workerLock } + } + Wait-For 'automatic non-crash recovery' { (Get-HotService).State -eq 'Running' } 150 + Wait-Output $localOut 'bounded-stop' 240 + Record 'controlled_err_scm_recovery' @{ lock_fault_removed = $true; recovered_without_manual_restart = $true } + [void] (Stop-HotService) + + # SMB is authenticated as an ephemeral non-admin Windows account. LocalService + # is deliberately not given an anonymous or guest share to manufacture success. + $random = [byte[]]::new(32); [Security.Cryptography.RandomNumberGenerator]::Fill($random) + $password = [Convert]::ToBase64String($random) + 'aA1!' + Write-Output "::add-mask::$password" + [void] (New-LocalUser -Name $user -Password (ConvertTo-SecureString $password -AsPlainText -Force) -PasswordNeverExpires -UserMayNotChangePassword -Description 'Disposable PdfCraft CI service account') + $accountCreated = $true + $userSid = (Get-LocalUser -Name $user).SID.Value + $export = Join-Path $script:TestRoot 'rights-before.inf' + [void] (Invoke-Program $secedit @('/export', '/cfg', $export, '/areas', 'USER_RIGHTS', '/quiet') 'export-service-rights') + $line = @(Get-Content -LiteralPath $export | Where-Object { $_ -match '^SeServiceLogonRight\s*=' }) + $originalRight = if ($line.Count) { ($line[0] -split '=', 2)[1].Trim() } else { '' } + $newRight = (@($originalRight -split ',' | Where-Object { $_ }) + "*$userSid") -join ',' + Set-ServiceLogonRight $newRight 'grant-service-right' + $privilegeChanged = $true + Set-PrivateDirectory $script:TestRoot @('S-1-5-19', $serviceSid, $userSid) + $prepared = Invoke-Tool 'prepare-account' 'hot_folder_service' @{ action = 'prepare-account'; executable = $companion; config = $script:Config } + Assert-That ($prepared.installed -and $prepared.account -eq 'NT AUTHORITY\LocalService') 'Automation preparation must retain the existing service account.' + Record 'prepare_account_automation' @{ installed = $prepared.installed; account_unchanged = $true } + # CIM passes the temporary secret directly to SCM; it is never a command-line + # argument, task JSON, installer property, source receipt, or log content. + $changed = Invoke-CimMethod -InputObject (Get-HotService) -MethodName Change -Arguments @{ StartName = $account; StartPassword = $password } + Assert-That ($changed.ReturnValue -eq 0) "SCM account configuration failed (code $($changed.ReturnValue))." + $password = $null + [void] (New-TestFolder 'smb') + $smbIn = New-TestFolder 'smb/in'; $smbOut = New-TestFolder 'smb/out' + New-TestShare + $smbTask = New-Task 'smb' "$sharePath\in" "$sharePath\out" + Save-Tasks @($localTask, $smbTask) 'configure-smb' + Put-Scan $smbIn 'authenticated-smb' + Start-HotService + Wait-Output $smbOut 'authenticated-smb' 120 + $sessions = @(Get-SmbSession | Where-Object { $_.ClientUserName -ieq $account }) + Assert-That ($sessions.Count -gt 0) 'No SMB session authenticated as the configured service account.' + Assert-That (Get-SmbShare -Name $shareName).EncryptData 'The test share must require SMB encryption.' + Record 'authenticated_smb_ocr' @{ account = $account; encrypted_share = $true; authenticated_sessions = $sessions.Count; source_unchanged = ((Get-FileHash -LiteralPath (Join-Path $smbIn 'authenticated-smb.pdf')).Hash -eq $scanHash) } + Remove-SmbShare -Name $shareName -Force + Put-Scan $smbIn 'after-share-outage' + Wait-For 'unavailable UNC recorded for affected task' { [bool] (Read-State).tasks.smb.error } 60 + Put-Scan $localIn 'during-share-outage' + Wait-Output $localOut 'during-share-outage' 90 + Assert-That ((Get-FileHash -LiteralPath (Join-Path $smbIn 'after-share-outage.pdf')).Hash -eq $scanHash) 'Unavailable share lost or changed the source.' + New-TestShare + Wait-Output $smbOut 'after-share-outage' 180 + Record 'smb_outage_isolated_and_auto_retried' @{ independent_local_task_completed = $true; no_manual_retry = $true; source_preserved = $true } + + # The real same-version MajorUpgrade must retain credentials kept by SCM. + # Changing ProductCode/PackageCode alone preserves the exact shipped payload. + [void] (Stop-HotService) + $beforeUpgradeConfig = (Get-FileHash -LiteralPath $script:Config).Hash + $beforeUpgradeAccount = (Get-HotService).StartName + $beforeUpgradeBinaries = @(Get-ChildItem -LiteralPath $appDirectory -File -Filter '*.exe' | ForEach-Object { "$($_.Name):$((Get-FileHash -LiteralPath $_.FullName).Hash)" }) + $upgrade = Join-Path $script:TestRoot 'same-version-upgrade-fixture.msi' + $productCodes.Add((New-UpgradeFixture $msi $upgrade)) + [void] (Invoke-Program $msiExe @('/i', $upgrade, '/qn', '/norestart', 'ALLUSERS=1', 'INSTALLHOTFOLDER=1', 'INSTALLDESKTOPSHORTCUT=0', "INSTALLFOLDER=$appDirectory", '/L*v', (Join-Path $script:Results 'upgrade-msi.log')) 'same-version-upgrade' 180 @(0, 3010)) + Wait-For 'upgraded service running' { (Get-HotService).State -eq 'Running' } 60 + Assert-That ((Get-HotService).StartName -ieq $beforeUpgradeAccount) 'Same-version MSI upgrade reset the service account.' + Assert-That ((Get-FileHash -LiteralPath $script:Config).Hash -eq $beforeUpgradeConfig) 'Upgrade changed user task settings.' + $afterUpgradeBinaries = @(Get-ChildItem -LiteralPath $appDirectory -File -Filter '*.exe' | ForEach-Object { "$($_.Name):$((Get-FileHash -LiteralPath $_.FullName).Hash)" }) + Assert-That (-not (Compare-Object $beforeUpgradeBinaries $afterUpgradeBinaries)) 'Upgrade fixture changed the real packaged executables.' + Put-Scan $smbIn 'after-upgrade' + Wait-Output $smbOut 'after-upgrade' 120 + Record 'same_version_major_upgrade_preserves_account' @{ same_real_payload = $true; test_only_product_and_package_guids = $true; account_unchanged = $true; tasks_unchanged = $true; authenticated_smb_after_upgrade = $true } + [void] (Stop-HotService) + Copy-Item -LiteralPath ([IO.Path]::ChangeExtension($script:Config, 'state.json')) -Destination (Join-Path $script:Results 'final-state.json') + $evidence.result = 'passed' +} catch { + $evidence.result = 'failed' + $evidence.error = $_.Exception.Message + throw +} finally { + $password = $null + $cleanupErrors = [Collections.Generic.List[string]]::new() + function Cleanup-Step([scriptblock] $Action) { + try { & $Action } catch { $cleanupErrors.Add($_.Exception.Message) } + } + Cleanup-Step { + $service = Get-HotService + if ($service) { + try { [void] (Stop-HotService) } catch { + if ($service.ProcessId -gt 0 -and (Get-Process -Id $service.ProcessId -ErrorAction SilentlyContinue).Path -eq $companion) { + Stop-Process -Id $service.ProcessId -Force + } + } + } + } + if ($installed -or (Test-Path -LiteralPath $marker)) { + foreach ($code in $productCodes) { + Cleanup-Step { + [void] (Invoke-Program $msiExe @('/x', $code, '/qn', '/norestart', '/L*v', (Join-Path $script:Results "uninstall-$($code.Trim('{}')).log")) ('uninstall-' + $code.Trim('{}')) 180 @(0, 1605, 1614, 3010)) + } + } + } + Cleanup-Step { + if (Get-SmbShare -Name $shareName -ErrorAction SilentlyContinue) { Remove-SmbShare -Name $shareName -Force } + } + Cleanup-Step { if ($privilegeChanged) { Set-ServiceLogonRight $originalRight 'restore-service-right' } } + Cleanup-Step { if ($accountCreated) { Remove-LocalUser -Name $user } } + Cleanup-Step { Assert-That (-not (Get-HotService)) 'Test service remains after uninstall.' } + Cleanup-Step { + if ((Test-Path -LiteralPath $marker) -and -not (Get-HotService)) { + Assert-That ((Get-Content -LiteralPath $marker -Raw) -ceq $token) 'Refusing to remove a directory without this run owner marker.' + [void] (Assert-SafePath $script:DataDirectory (Join-Path $env:ProgramData 'PdfCraft')) + Remove-Item -LiteralPath $script:DataDirectory -Recurse -Force + } + } + $evidence.cleanup = @{ errors = @($cleanupErrors); service_removed = (-not (Get-HotService)); ephemeral_account_removed = (-not (Get-LocalUser -Name $user -ErrorAction SilentlyContinue)); share_removed = (-not (Get-SmbShare -Name $shareName -ErrorAction SilentlyContinue)) } + if ($cleanupErrors.Count) { $evidence.result = 'failed' } + Write-Json (Join-Path $script:Results 'report.json') $evidence + # Keep private working files on failure for the disposable runner's diagnosis; + # only the explicitly selected result/log folder is uploaded by the workflow. + if ($evidence.result -eq 'passed') { + [void] (Assert-SafePath $script:TestRoot $env:RUNNER_TEMP) + Remove-Item -LiteralPath $script:TestRoot -Recurse -Force + } + if ($cleanupErrors.Count) { throw "Acceptance cleanup failed: $($cleanupErrors -join '; ')" } +} diff --git a/packaging/server-tests/actionlint.yaml b/packaging/server-tests/actionlint.yaml new file mode 100644 index 0000000000..ac502c861d --- /dev/null +++ b/packaging/server-tests/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - pdfcraft-server-2019-disposable diff --git a/packaging/server-tests/check.ps1 b/packaging/server-tests/check.ps1 new file mode 100644 index 0000000000..3114f0bfc0 --- /dev/null +++ b/packaging/server-tests/check.ps1 @@ -0,0 +1,64 @@ +# Safe local checks only. Never run the installer acceptance body here. +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot/common.ps1" +foreach ($file in Get-ChildItem -LiteralPath $PSScriptRoot -Filter '*.ps1') { + $tokens = $null; $errors = $null + [void] [Management.Automation.Language.Parser]::ParseFile($file.FullName, [ref] $tokens, [ref] $errors) + Assert-That (-not $errors) ("PowerShell syntax: " + ($errors -join '; ')) +} +$boundary = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) +[void] (Assert-SafePath (Join-Path $boundary 'packaging/server-tests') $boundary) +$rejected = $false +try { [void] (Assert-SafePath ($boundary + '-sibling/file') $boundary) } catch { $rejected = $true } +Assert-That $rejected 'Sibling-prefix confinement must reject paths outside the test root.' + +$keys = @('GITHUB_EVENT_NAME', 'INSTALLER_RUN_ID', 'EXPECTED_SHA', 'GITHUB_SHA', 'INCLUDE_2019', + 'CONFIRM_2019', 'ARTIFACT_NAME', 'GITHUB_API_URL', 'GITHUB_REPOSITORY', 'GITHUB_OUTPUT', 'GH_TOKEN') +$saved = @{} +foreach ($key in $keys) { $saved[$key] = [Environment]::GetEnvironmentVariable($key) } +$output = [IO.Path]::GetTempFileName() +$previousCase = Get-Variable -Name PdfcraftServerProvenanceCase -Scope Global -ErrorAction SilentlyContinue +try { + $env:GITHUB_EVENT_NAME = 'workflow_dispatch'; $env:INSTALLER_RUN_ID = '123' + $env:EXPECTED_SHA = 'a' * 40; $env:GITHUB_SHA = $env:EXPECTED_SHA + $env:INCLUDE_2019 = 'false'; $env:CONFIRM_2019 = '' + $env:ARTIFACT_NAME = 'pdfcraft-preview-windows-x64' + $env:GITHUB_API_URL = 'https://api.invalid'; $env:GITHUB_REPOSITORY = 'test/pdfcraft' + $env:GITHUB_OUTPUT = $output; $env:GH_TOKEN = 'synthetic-not-a-token' + $global:PdfcraftServerProvenanceCase = 'good' + function Invoke-RestMethod($Uri, $Headers) { + if ($Uri -like '*/artifacts?*') { + $sha = if ($global:PdfcraftServerProvenanceCase -eq 'artifact-sha') { 'b' * 40 } else { $env:EXPECTED_SHA } + return @{ total_count = 1; artifacts = @(@{ name = $env:ARTIFACT_NAME; expired = $false; id = 42; workflow_run = @{ head_sha = $sha } }) } + } + if ($Uri -like '*/workflows/*') { + return @{ path = if ($global:PdfcraftServerProvenanceCase -eq 'workflow') { '.github/workflows/other.yml' } else { '.github/workflows/preview-installers.yml' } } + } + return @{ + status = 'completed'; conclusion = if ($global:PdfcraftServerProvenanceCase -eq 'failed') { 'failure' } else { 'success' } + head_sha = $env:EXPECTED_SHA; workflow_id = 7 + head_repository = @{ full_name = if ($global:PdfcraftServerProvenanceCase -eq 'fork') { 'other/pdfcraft' } else { $env:GITHUB_REPOSITORY } } + event = if ($global:PdfcraftServerProvenanceCase -eq 'pull-request') { 'pull_request' } else { 'workflow_dispatch' } + } + } + & "$PSScriptRoot/verify-provenance.ps1" | Out-Null + Assert-That ((Get-Content -LiteralPath $output -Raw) -match 'artifact_id=42') 'Matching trusted artifact should pass.' + foreach ($invalid in @('failed', 'fork', 'workflow', 'pull-request', 'artifact-sha')) { + $global:PdfcraftServerProvenanceCase = $invalid; $rejected = $false + try { & "$PSScriptRoot/verify-provenance.ps1" | Out-Null } catch { $rejected = $true } + Assert-That $rejected "Unsafe artifact accepted: $invalid" + } + $global:PdfcraftServerProvenanceCase = 'good'; $env:INCLUDE_2019 = 'true'; $rejected = $false + try { & "$PSScriptRoot/verify-provenance.ps1" | Out-Null } catch { $rejected = $true } + Assert-That $rejected 'Unconfirmed Server 2019 dispatch must fail.' + $env:INCLUDE_2019 = 'false'; $env:GITHUB_EVENT_NAME = 'pull_request'; $rejected = $false + try { & "$PSScriptRoot/verify-provenance.ps1" | Out-Null } catch { $rejected = $true } + Assert-That $rejected 'Automatic parent event must fail.' +} finally { + Remove-Item -LiteralPath $output -Force + foreach ($key in $keys) { [Environment]::SetEnvironmentVariable($key, $saved[$key]) } + Remove-Item Function:Invoke-RestMethod + if ($previousCase) { $global:PdfcraftServerProvenanceCase = $previousCase.Value } + else { Remove-Variable -Name PdfcraftServerProvenanceCase -Scope Global } +} +Write-Output 'ok: parser, safe-path confinement and eight provenance scenarios; no service or installer executed' diff --git a/packaging/server-tests/common.ps1 b/packaging/server-tests/common.ps1 new file mode 100644 index 0000000000..5bc3d407fd --- /dev/null +++ b/packaging/server-tests/common.ps1 @@ -0,0 +1,166 @@ +$ErrorActionPreference = 'Stop' + +function Assert-That([bool] $Condition, [string] $Message) { + if (-not $Condition) { throw $Message } +} + +function Write-Json([string] $Path, $Value) { + [IO.File]::WriteAllText($Path, ($Value | ConvertTo-Json -Depth 40), [Text.UTF8Encoding]::new($false)) +} + +function Assert-SafePath([string] $Path, [string] $AllowedRoot) { + $absolute = [IO.Path]::GetFullPath($Path) + $boundary = [IO.Path]::GetFullPath($AllowedRoot).TrimEnd('\') + Assert-That ($absolute -eq $boundary -or $absolute.StartsWith($boundary + '\', [StringComparison]::OrdinalIgnoreCase)) "Unsafe test path: $absolute" + $cursor = $absolute + while ($cursor -and $cursor.Length -ge $boundary.Length) { + if (Test-Path -LiteralPath $cursor) { + Assert-That (-not ((Get-Item -LiteralPath $cursor -Force).Attributes -band [IO.FileAttributes]::ReparsePoint)) "Test path traverses a reparse point: $cursor" + } + $cursor = [IO.Path]::GetDirectoryName($cursor) + } + return $absolute +} + +function Invoke-Program([string] $Exe, [string[]] $Arguments, [string] $Name, [int] $TimeoutSeconds = 120, [int[]] $SuccessCodes = @(0)) { + $start = [Diagnostics.ProcessStartInfo]::new() + $start.FileName = $Exe + $start.WorkingDirectory = $script:TestRoot + $start.UseShellExecute = $false + $start.CreateNoWindow = $true + $start.WindowStyle = [Diagnostics.ProcessWindowStyle]::Hidden + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + $start.StandardOutputEncoding = [Text.Encoding]::UTF8 + $start.StandardErrorEncoding = [Text.Encoding]::UTF8 + foreach ($argument in $Arguments) { [void] $start.ArgumentList.Add($argument) } + $process = [Diagnostics.Process]::Start($start) + try { + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + $process.Kill($true) + throw "$Name exceeded $TimeoutSeconds seconds" + } + $output = $stdout.GetAwaiter().GetResult() + $errorText = $stderr.GetAwaiter().GetResult() + [IO.File]::WriteAllText((Join-Path $script:Results "$Name.log"), $output + $errorText, [Text.UTF8Encoding]::new($false)) + Assert-That ($process.ExitCode -in $SuccessCodes) "$Name exited with code $($process.ExitCode); see $Name.log" + return $output + } finally { $process.Dispose() } +} + +function Wait-For([string] $What, [scriptblock] $Predicate, [int] $Seconds = 120) { + $timer = [Diagnostics.Stopwatch]::StartNew() + $lastError = '' + while ($timer.Elapsed.TotalSeconds -lt $Seconds) { + try { if (& $Predicate) { return } } catch { $lastError = $_.Exception.Message } + Start-Sleep -Milliseconds 500 + } + throw "Timed out waiting for $What ($Seconds seconds). $lastError" +} + +function Get-HotService { + Get-CimInstance Win32_Service -Filter "Name='PdfCraftHotFolder'" +} + +function Stop-HotService([int] $Seconds = 35) { + $watch = [Diagnostics.Stopwatch]::StartNew() + $service = Get-HotService + if ($service -and $service.State -ne 'Stopped') { + $request = Invoke-CimMethod -InputObject $service -MethodName StopService + Assert-That ($request.ReturnValue -in @(0, 5, 6)) "Service stop request failed: $($request.ReturnValue)" + Wait-For 'service stopped' { (Get-HotService).State -eq 'Stopped' } $Seconds + } + return [Math]::Round($watch.Elapsed.TotalSeconds, 3) +} + +function Start-HotService { + $request = Invoke-CimMethod -InputObject (Get-HotService) -MethodName StartService + Assert-That ($request.ReturnValue -in @(0, 10)) "Service start request failed: $($request.ReturnValue)" + Wait-For 'service running' { (Get-HotService).State -eq 'Running' } 40 +} + +function Invoke-CliScript([string] $Name, [array] $Steps) { + $path = Join-Path $script:TestRoot "$Name.json" + Write-Json $path $Steps + Invoke-Program $script:Cli @('run', '--script', $path) $Name 180 +} + +function Invoke-Tool([string] $Name, [string] $Tool, $Arguments) { + $raw = Invoke-CliScript $Name @(@{ tool = $Tool; args = $Arguments }) + return $raw | ConvertFrom-Json +} + +function New-Task([string] $Id, [string] $InputFolder, [string] $OutputFolder, [string] $Originals = 'keep') { + return @{ + id = $Id; name = $Id; input = $InputFolder; output = $OutputFolder + enabled = $true; interval_seconds = 1; settle_seconds = 1; originals = $Originals + settings = @{ language = 'en'; recognizer = 'modern'; recognize = $true; dpi = 150; auto_rotate = $false; deskew = $false; auto_crop = $false } + output_plan = @{ steps = @(@{ format = 'pdf' }, @{ format = 'txt' }) } + } +} + +function Save-Tasks([array] $Tasks, [string] $Label) { + [void] (Invoke-Tool $Label 'hot_folder_configure' @{ config = $script:Config; settings = @{ tasks = $Tasks } }) +} + +function Read-State { + Get-Content -LiteralPath ([IO.Path]::ChangeExtension($script:Config, 'state.json')) -Raw | ConvertFrom-Json +} + +function Wait-Output([string] $Directory, [string] $Stem, [int] $Seconds = 180) { + Wait-For "searchable output $Stem" { + $text = Join-Path $Directory "$Stem.txt" + (Test-Path -LiteralPath (Join-Path $Directory "$Stem.pdf")) -and (Test-Path -LiteralPath $text) -and + ((Get-Content -LiteralPath $text -Raw) -match 'searchable output') + } $Seconds +} + +function Set-PrivateDirectory([string] $Path, [string[]] $ExtraSids = @()) { + $acl = [Security.AccessControl.DirectorySecurity]::new() + $acl.SetAccessRuleProtection($true, $false) + foreach ($id in @('S-1-5-18', 'S-1-5-32-544') + $ExtraSids) { + $sid = [Security.Principal.SecurityIdentifier]::new($id) + $rule = [Security.AccessControl.FileSystemAccessRule]::new($sid, 'FullControl', 'ContainerInherit,ObjectInherit', 'None', 'Allow') + $acl.AddAccessRule($rule) + } + Set-Acl -LiteralPath $Path -AclObject $acl +} + +function Get-ProductCode([string] $Msi) { + $installer = New-Object -ComObject WindowsInstaller.Installer + $database = $installer.OpenDatabase($Msi, 0) + try { + $view = $database.OpenView('SELECT `Value` FROM `Property` WHERE `Property` = ''ProductCode''') + [void] $view.Execute() + return $view.Fetch().StringData(1) + } finally { + if ($view) { [void] $view.Close() } + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($database) + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($installer) + } +} + +function New-UpgradeFixture([string] $Msi, [string] $Destination) { + # Identical real packaged binaries; changed MSI identities exercise a genuine + # same-version MajorUpgrade, not REINSTALL/repair. Never publish this fixture. + Copy-Item -LiteralPath $Msi -Destination $Destination + $installer = New-Object -ComObject WindowsInstaller.Installer + $database = $installer.OpenDatabase($Destination, 1) + try { + $product = '{' + [guid]::NewGuid().ToString().ToUpperInvariant() + '}' + $view = $database.OpenView('UPDATE `Property` SET `Value` = ''' + $product + ''' WHERE `Property` = ''ProductCode''') + [void] $view.Execute() + [void] $view.Close() + $summary = $database.SummaryInformation(20) + $summary.Property(9) = '{' + [guid]::NewGuid().ToString().ToUpperInvariant() + '}' + $summary.Persist() + $database.Commit() + return $product + } finally { + if ($summary) { [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($summary) } + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($database) + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($installer) + } +} diff --git a/packaging/server-tests/verify-provenance.ps1 b/packaging/server-tests/verify-provenance.ps1 new file mode 100644 index 0000000000..a843a0d25e --- /dev/null +++ b/packaging/server-tests/verify-provenance.ps1 @@ -0,0 +1,30 @@ +# Pure GitHub metadata validation. No installer or downloaded code is executed. +$ErrorActionPreference = 'Stop' +if ($env:GITHUB_EVENT_NAME -ne 'workflow_dispatch') { throw 'Server acceptance is manual only.' } +if ($env:INSTALLER_RUN_ID -notmatch '^[0-9]+$') { throw 'A numeric installer run ID is required.' } +if ($env:EXPECTED_SHA -notmatch '^[a-f0-9]{40}$' -or $env:EXPECTED_SHA -cne $env:GITHUB_SHA) { + throw 'Installer source SHA must exactly match the explicitly selected workflow ref.' +} +if ($env:INCLUDE_2019 -eq 'true' -and $env:CONFIRM_2019 -cne 'DISPOSABLE SERVER 2019') { + throw 'The self-hosted Server 2019 test needs the exact disposable-host confirmation.' +} +if ($env:ARTIFACT_NAME -notmatch '^[A-Za-z0-9_.-]+$') { throw 'Invalid artifact name.' } +$headers = @{ Authorization = "Bearer $env:GH_TOKEN"; Accept = 'application/vnd.github+json'; 'X-GitHub-Api-Version' = '2022-11-28' } +$api = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions" +$run = Invoke-RestMethod -Uri "$api/runs/$env:INSTALLER_RUN_ID" -Headers $headers +if ($run.status -ne 'completed' -or $run.conclusion -ne 'success') { throw 'Installer workflow must have completed successfully.' } +if ($run.head_sha -cne $env:EXPECTED_SHA -or $run.head_repository.full_name -cne $env:GITHUB_REPOSITORY) { + throw 'Installer artifact is from a different commit or repository.' +} +if ($run.event -notin @('push', 'workflow_dispatch', 'release')) { throw 'PR and other untrusted installer triggers are refused.' } +$workflow = Invoke-RestMethod -Uri "$api/workflows/$($run.workflow_id)" -Headers $headers +if ($workflow.path -notin @('.github/workflows/preview-installers.yml', '.github/workflows/release.yml')) { + throw 'Only the repository installer workflows are accepted.' +} +$artifacts = Invoke-RestMethod -Uri "$api/runs/$env:INSTALLER_RUN_ID/artifacts?per_page=100" -Headers $headers +if ($artifacts.total_count -gt 100) { throw 'Artifact list exceeds the bounded verifier.' } +$found = @($artifacts.artifacts | Where-Object { $_.name -ceq $env:ARTIFACT_NAME -and -not $_.expired }) +if ($found.Count -ne 1) { throw 'Exactly one unexpired matching installer artifact is required.' } +if ($found[0].workflow_run.head_sha -cne $env:EXPECTED_SHA) { throw 'Artifact commit differs from the workflow commit.' } +"artifact_id=$($found[0].id)" | Out-File -LiteralPath $env:GITHUB_OUTPUT -Encoding utf8 -Append +Write-Output "Verified installer run $env:INSTALLER_RUN_ID, SHA $env:EXPECTED_SHA, artifact $($found[0].id)." diff --git a/packaging/service-tests/windows-msi.ps1 b/packaging/service-tests/windows-msi.ps1 index 80545d25fe..5db8ca3163 100644 --- a/packaging/service-tests/windows-msi.ps1 +++ b/packaging/service-tests/windows-msi.ps1 @@ -19,6 +19,11 @@ if ($LASTEXITCODE -ne 0) { throw 'WiX extension installation failed' } if ($LASTEXITCODE -ne 0) { throw 'WiX compilation failed' } & pwsh -NoProfile -File packaging/windows/test-msi.ps1 $msi if ($LASTEXITCODE -ne 0) { throw 'Compiled MSI validation failed' } +# A harmless PE overlay makes the upgrade payload observably different, without +# depending on the probe executable having a version resource. +$probeStream = [IO.File]::Open("$stage/pdfcraft-hot-folder.exe", [IO.FileMode]::Append) +try { $probeStream.Write([Text.Encoding]::ASCII.GetBytes('synthetic upgrade payload')) } finally { $probeStream.Dispose() } +$upgradeHash = (Get-FileHash -LiteralPath "$stage/pdfcraft-hot-folder.exe" -Algorithm SHA256).Hash & wix build packaging/windows/pdfcraft.wxs packaging/windows/installer-ui.wxs -arch x64 -ext WixToolset.Util.wixext -d Version=0.6.1 -d "BinDir=$stage" -d "ModelsDir=$stage/models" -d 'IconPath=assets/app-icon/pdfcraft.ico' -o $upgrade if ($LASTEXITCODE -ne 0) { throw 'Upgrade MSI compilation failed' } function Invoke-Msi([string] $Options, [string] $Name) { @@ -30,6 +35,15 @@ function Invoke-Msi([string] $Options, [string] $Name) { } } $data = Join-Path $env:ProgramData 'PdfCraft/HotFolder' +function Wait-NewServiceProcess([uint32] $Previous, [int] $Seconds = 100) { + $deadline = [DateTime]::UtcNow.AddSeconds($Seconds) + do { + $current = Get-CimInstance Win32_Service -Filter "Name='PdfCraftHotFolder'" + if ($current.State -eq 'Running' -and $current.ProcessId -ne 0 -and $current.ProcessId -ne $Previous) { return $current.ProcessId } + Start-Sleep -Milliseconds 300 + } while ([DateTime]::UtcNow -lt $deadline) + throw 'SCM did not automatically recover the service within the deadline' +} try { Invoke-Msi "/i `"$msi`"" 'default' if (Get-Service PdfCraftHotFolder -ErrorAction SilentlyContinue) { throw 'Default install unexpectedly installed the service' } @@ -40,15 +54,45 @@ try { } for ($i = 0; $i -lt 40 -and -not (Test-Path -LiteralPath "$data/tasks.heartbeat"); $i++) { Start-Sleep -Milliseconds 250 } if (-not (Test-Path -LiteralPath "$data/tasks.heartbeat")) { throw 'Installed service cannot write its protected task directory' } + $registered = Get-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Services\PdfCraftHotFolder' + if ($registered.ServiceSidType -ne 1 -or $registered.FailureActionsOnNonCrashFailures -ne 1) { + throw 'MSI did not prepare a service SID and recovery for controlled errors' + } + $identity = [System.Security.Principal.NTAccount]::new('NT SERVICE', 'PdfCraftHotFolder') + $sid = $identity.Translate([System.Security.Principal.SecurityIdentifier]).Value + if ($sid -ne 'S-1-5-80-4013549229-3107762393-4025766903-2826249871-1347187068') { throw 'Installer service SID is incorrect' } + # Recovery must continue after the old two-restart limit, and also after an + # explicit SERVICE_STOPPED error. All faults stay on this disposable runner. + $servicePid = [uint32] $service.ProcessId + 1..4 | ForEach-Object { + Stop-Process -Id $servicePid -Force + $servicePid = Wait-NewServiceProcess $servicePid + } + Set-Content -LiteralPath "$data/tasks.fail" -Value 'synthetic controlled failure' + $servicePid = Wait-NewServiceProcess $servicePid + Stop-Service PdfCraftHotFolder + Start-Sleep -Seconds 65 + if ((Get-Service PdfCraftHotFolder).Status -ne 'Stopped') { throw 'An administrator stop must not trigger recovery' } + # A different Windows-managed identity requires no test password. The final + # server suite additionally covers an authenticated SMB account and same-version upgrade. + & sc.exe config PdfCraftHotFolder obj= 'NT AUTHORITY\NetworkService' + if ($LASTEXITCODE -ne 0) { throw 'Cannot configure the synthetic alternate service account' } + Start-Service PdfCraftHotFolder + $heartbeatBefore = (Get-Item -LiteralPath "$data/tasks.heartbeat").LastWriteTimeUtc + Start-Sleep -Seconds 2 + if ((Get-Item -LiteralPath "$data/tasks.heartbeat").LastWriteTimeUtc -le $heartbeatBefore) { throw 'Alternate account cannot write through the service SID' } Set-Content -LiteralPath "$data/keep-user-tasks.txt" -Value 'synthetic task preservation fixture' # No INSTALLHOTFOLDER override: a major upgrade must preserve the opt-in choice. Invoke-Msi "/i `"$upgrade`"" 'upgrade' $installedMsi = $upgrade $service = Get-CimInstance Win32_Service -Filter "Name='PdfCraftHotFolder'" - if ($service.State -ne 'Running' -or $service.StartMode -ne 'Auto' -or $service.StartName -ne 'NT AUTHORITY\LocalService') { + if ($service.State -ne 'Running' -or $service.StartMode -ne 'Auto' -or $service.StartName -ne 'NT AUTHORITY\NetworkService') { throw 'Upgrade did not preserve the optional automatic service and its account' } if (-not (Test-Path -LiteralPath "$data/keep-user-tasks.txt")) { throw 'Upgrade deleted task data' } + if ((Get-FileHash -LiteralPath "$env:ProgramFiles/PdfCraft/pdfcraft-hot-folder.exe" -Algorithm SHA256).Hash -ne $upgradeHash) { + throw 'Upgrade retained the old executable instead of the new payload' + } Invoke-Msi "/i `"$installedMsi`" REMOVE=HotFolder" 'remove-companion' if (Get-Service PdfCraftHotFolder -ErrorAction SilentlyContinue) { throw 'Removing the component left its service installed' } if (-not (Test-Path -LiteralPath "$data/keep-user-tasks.txt")) { throw 'Removing the component deleted task data' } diff --git a/packaging/windows/pdfcraft.wxs b/packaging/windows/pdfcraft.wxs index 84b1f3391e..77f0b20b4e 100644 --- a/packaging/windows/pdfcraft.wxs +++ b/packaging/windows/pdfcraft.wxs @@ -25,8 +25,13 @@ Message="PdfCraft must be installed for all users. Run setup with administrator privileges and ALLUSERS=1; per-user installation is not supported." /> - + + + + + + + + @@ -85,14 +100,14 @@ Account="NT AUTHORITY\LocalService" Arguments="--service --config "[HotFolderData]tasks.json""> + ThirdFailureActionType="restart" ResetPeriodInDays="1" RestartServiceDelayInSeconds="60" />     - + @@ -101,7 +116,7 @@ - + diff --git a/packaging/windows/test-msi.ps1 b/packaging/windows/test-msi.ps1 index 947a3f68e2..ff335693bf 100644 --- a/packaging/windows/test-msi.ps1 +++ b/packaging/windows/test-msi.ps1 @@ -164,9 +164,17 @@ $control = Read-Row 'SELECT `Event`, `Wait` FROM `ServiceControl` WHERE `Name` = Assert-Equal $control[0] '163' 'Start on install, stop on upgrade/removal, delete on uninstall' Assert-Equal $control[1] '1' 'Wait for graceful service stop' $acl = Read-Row 'SELECT `SDDLText` FROM `MsiLockPermissionsEx` WHERE `LockObject` = ''HotFolderData''' 1 -Assert-Equal $acl[0] 'D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;S-1-5-19)(A;OICI;GRGX;;;BU)' 'Protected task directory' +Assert-Equal $acl[0] 'D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;S-1-5-19)(A;OICI;FA;;;S-1-5-80-4013549229-3107762393-4025766903-2826249871-1347187068)(A;OICI;GRGX;;;BU)' 'Protected task directory' $secrets = Read-Row 'SELECT `SDDLText` FROM `MsiLockPermissionsEx` WHERE `LockObject` = ''HotFolderSecrets''' 1 -Assert-Equal $secrets[0] 'D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;S-1-5-19)' 'Credentials must not be readable by ordinary users' +Assert-Equal $secrets[0] 'D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;S-1-5-19)(A;OICI;FA;;;S-1-5-80-4013549229-3107762393-4025766903-2826249871-1347187068)' 'Credentials must not be readable by ordinary users' +$prepare = Read-Row 'SELECT `Type`, `Source`, `Target` FROM `CustomAction` WHERE `Action` = ''PrepareHotFolderService''' 3 +Assert-Equal $prepare[1] 'PdfcraftHotFolderExe' 'Prepare service uses installed executable' +Assert-Equal $prepare[2] '--service-action prepare-account --config "[HotFolderData]tasks.json"' 'Prepare service has no passwords' +if (([int] $prepare[0] -band 3072) -ne 3072) { throw 'Service preparation must be deferred and elevated' } +$prepareSequence = Read-Row 'SELECT `Sequence` FROM `InstallExecuteSequence` WHERE `Action` = ''PrepareHotFolderService''' 1 +$installServices = Read-Row 'SELECT `Sequence` FROM `InstallExecuteSequence` WHERE `Action` = ''InstallServices''' 1 +$startServices = Read-Row 'SELECT `Sequence` FROM `InstallExecuteSequence` WHERE `Action` = ''StartServices''' 1 +if ([int] $prepareSequence[0] -le [int] $installServices[0] -or [int] $prepareSequence[0] -ge [int] $startServices[0]) { throw 'Service preparation must run between registration and start' } $shortcut = Read-Row 'SELECT `Target`, `Arguments` FROM `Shortcut` WHERE `Shortcut` = ''HotFolderStartMenuShortcut''' 2 Assert-Equal $shortcut[0] '[#PdfcraftHotFolderExe]' 'Hot Folder shortcut executable' Assert-Equal $shortcut[1] '--config "[HotFolderData]tasks.json"' 'GUI and service use the same tasks' diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 97e0343280..fbc4019d15 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -1718,7 +1718,7 @@ milestone = "M3" status = "partial" commands = ["help.check_updates"] tools = [] -evidence = ["crates/ui-egui/tests/updates.rs::a_newer_release_is_offered_for_download", "crates/ui-egui/tests/updates.rs::nothing_is_asked_until_the_user_checks", "apps/pdfcraft/src/updates.rs::answers_are_read_and_only_our_release_pages_are_offered"] +evidence = ["crates/ui-egui/tests/release_checks.rs::a_newer_release_is_offered_for_download", "crates/ui-egui/tests/release_checks.rs::nothing_is_asked_until_the_user_checks", "apps/pdfcraft/src/updates.rs::answers_are_read_and_only_our_release_pages_are_offered"] notes = "asks GitHub Releases on request and offers the release page. By the owner's decision there is no check at start and no in-place install (Acrobat has both)" [[feature]] From 391a74d09d8c889ac663d688feb20b5758211dda Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:38:59 +0200 Subject: [PATCH 07/19] Localize Windows and macOS installers using the primary system language --- .github/workflows/installer-languages.yml | 45 +++++ ATTRIBUTION.md | 16 +- ATTRIBUTION.toml | 154 ++++++++++++++++++ packaging/installer-languages.json | 44 +++++ packaging/installer-languages.txt | 90 ++++++++++ packaging/macos/Distribution.xml.in | 2 +- packaging/macos/package.sh | 4 +- packaging/macos/prepare-resources.sh | 35 ++++ .../resources/de.lproj/Localizable.strings | 4 + .../resources/en.lproj/Localizable.strings | 4 + .../resources/es.lproj/Localizable.strings | 4 + .../resources/fr.lproj/Localizable.strings | 4 + .../resources/hu.lproj/Localizable.strings | 4 + .../resources/it.lproj/Localizable.strings | 4 + .../resources/nl.lproj/Localizable.strings | 4 + packaging/macos/test-localization.applescript | 32 ++++ packaging/macos/test-localized-pkg.sh | 34 ++++ packaging/service-tests/macos-pkg.sh | 3 +- packaging/service-tests/windows-msi.ps1 | 4 +- packaging/test-installer-languages.py | 67 ++++++++ packaging/windows/build-localized.ps1 | 73 +++++++++ packaging/windows/embed-transforms.ps1 | 27 +++ packaging/windows/installer-languages.ps1 | 27 +++ packaging/windows/installer-ui.wxs | 108 ++++++------ packaging/windows/locales/de.wxl | 44 +++++ packaging/windows/locales/en.wxl | 44 +++++ packaging/windows/locales/es.wxl | 44 +++++ packaging/windows/locales/fr.wxl | 44 +++++ packaging/windows/locales/hu.wxl | 44 +++++ packaging/windows/locales/it.wxl | 44 +++++ packaging/windows/locales/nl.wxl | 44 +++++ packaging/windows/package.ps1 | 20 +-- packaging/windows/pdfcraft.wxs | 48 +++--- packaging/windows/setup.wxs | 18 ++ packaging/windows/test-language-build.ps1 | 19 +++ packaging/windows/test-localized.ps1 | 63 +++++++ packaging/windows/test-setup.ps1 | 48 ++++++ 37 files changed, 1222 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/installer-languages.yml create mode 100644 packaging/installer-languages.json create mode 100644 packaging/installer-languages.txt create mode 100644 packaging/macos/prepare-resources.sh create mode 100644 packaging/macos/resources/de.lproj/Localizable.strings create mode 100644 packaging/macos/resources/en.lproj/Localizable.strings create mode 100644 packaging/macos/resources/es.lproj/Localizable.strings create mode 100644 packaging/macos/resources/fr.lproj/Localizable.strings create mode 100644 packaging/macos/resources/hu.lproj/Localizable.strings create mode 100644 packaging/macos/resources/it.lproj/Localizable.strings create mode 100644 packaging/macos/resources/nl.lproj/Localizable.strings create mode 100644 packaging/macos/test-localization.applescript create mode 100644 packaging/macos/test-localized-pkg.sh create mode 100644 packaging/test-installer-languages.py create mode 100644 packaging/windows/build-localized.ps1 create mode 100644 packaging/windows/embed-transforms.ps1 create mode 100644 packaging/windows/installer-languages.ps1 create mode 100644 packaging/windows/locales/de.wxl create mode 100644 packaging/windows/locales/en.wxl create mode 100644 packaging/windows/locales/es.wxl create mode 100644 packaging/windows/locales/fr.wxl create mode 100644 packaging/windows/locales/hu.wxl create mode 100644 packaging/windows/locales/it.wxl create mode 100644 packaging/windows/locales/nl.wxl create mode 100644 packaging/windows/setup.wxs create mode 100644 packaging/windows/test-language-build.ps1 create mode 100644 packaging/windows/test-localized.ps1 create mode 100644 packaging/windows/test-setup.ps1 diff --git a/.github/workflows/installer-languages.yml b/.github/workflows/installer-languages.yml new file mode 100644 index 0000000000..76d3f61627 --- /dev/null +++ b/.github/workflows/installer-languages.yml @@ -0,0 +1,45 @@ +name: Installer languages +on: + workflow_dispatch: + pull_request: + paths: + - 'packaging/**' + - '.github/workflows/installer-languages.yml' + push: + branches: [main, codex/installer-languages] + paths: + - 'packaging/**' + - '.github/workflows/installer-languages.yml' +permissions: + contents: read +jobs: + windows: + name: Offline Setup and MSI (${{ matrix.arch }}) + runs-on: windows-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + arch: [x64, arm64, x86] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + - name: Install pinned WiX build tool + run: dotnet tool install --global wix --version 5.0.2 + - name: Catalog parity and codepages + run: python packaging/test-installer-languages.py + - name: Build and inspect all seven languages (no installation) + shell: pwsh + run: ./packaging/windows/test-language-build.ps1 -Arch ${{ matrix.arch }} -Work "$env:RUNNER_TEMP/installer-language-fixture" + macos: + name: Native localized PKG + runs-on: macos-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Catalog parity and codepages + run: python3 packaging/test-installer-languages.py + - name: Build PKG and inspect native locale resolution (no installation) + run: bash packaging/macos/test-localized-pkg.sh "$RUNNER_TEMP/installer-language-fixture" diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 2772c9fea5..7fc54f0faa 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -4,7 +4,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, with its author, source and licence. The policy is in [AGENTS.md](AGENTS.md) §1. The machine-readable list, with SHA-256 hashes, is [ATTRIBUTION.toml](ATTRIBUTION.toml). Licence texts are kept beside the assets and summarised in [NOTICE](NOTICE). -## In this repository (264) +## In this repository (278) | Asset | Title | Author | Licence | Source | Used for | |---|---|---|---|---|---| @@ -272,6 +272,20 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `packaging/macos/dmg/background.svg` | PdfCraft macOS DMG window background | @XusBadia | MIT OR Apache-2.0 | Original work for this repository, derived from the PdfCraft app icon (assets/app-icon/pdfcraft.svg, referenced via an SVG image element, not copied) | Finder window of the macOS DMG; rendered to background.tiff by packaging/macos/dmg/generate.py | | `packaging/macos/dmg/background.tiff` | PdfCraft macOS DMG window background (rendered) | @XusBadia | MIT OR Apache-2.0 | Rendered from packaging/macos/dmg/background.svg by packaging/macos/dmg/generate.py | 1x + 2x HiDPI 16-colour palette TIFF copied into the macOS DMG by packaging/macos/package.sh; includes the app icon and the free LittleCMS built-in sRGB profile | | `crates/sign/tests/data/x509-rsa-sha1.pdf` | One-page PDF with a legacy adbe.x509.rsa_sha1 signature by the RSA test key | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: hand-written PDF objects; signature made with `openssl dgst -sha1 -sign` over its byte ranges (see crates/sign/tests/data/README.md) | Signature validation tests (crates/sign/tests/pdf.rs): the legacy adbe.x509.rsa_sha1 format | +| `packaging/windows/locales/de.wxl` | PdfCraft installer translation catalog (de) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/windows/locales/en.wxl` | PdfCraft installer translation catalog (en) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/windows/locales/es.wxl` | PdfCraft installer translation catalog (es) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/windows/locales/fr.wxl` | PdfCraft installer translation catalog (fr) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/windows/locales/hu.wxl` | PdfCraft installer translation catalog (hu) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/windows/locales/it.wxl` | PdfCraft installer translation catalog (it) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/windows/locales/nl.wxl` | PdfCraft installer translation catalog (nl) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/macos/resources/de.lproj/Localizable.strings` | PdfCraft installer translation catalog (de) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/macos/resources/en.lproj/Localizable.strings` | PdfCraft installer translation catalog (en) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/macos/resources/es.lproj/Localizable.strings` | PdfCraft installer translation catalog (es) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/macos/resources/fr.lproj/Localizable.strings` | PdfCraft installer translation catalog (fr) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/macos/resources/hu.lproj/Localizable.strings` | PdfCraft installer translation catalog (hu) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/macos/resources/it.lproj/Localizable.strings` | PdfCraft installer translation catalog (it) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | +| `packaging/macos/resources/nl.lproj/Localizable.strings` | PdfCraft installer translation catalog (nl) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources | Native installer UI; English also supplies generated fallback locale catalogs | ## Compiled in through dependencies (6) diff --git a/ATTRIBUTION.toml b/ATTRIBUTION.toml index 843548d81a..6a7c3a3bdf 100644 --- a/ATTRIBUTION.toml +++ b/ATTRIBUTION.toml @@ -3324,3 +3324,157 @@ licence_sha256 = "3840c5c0c61c294264d2dd77b8777be6ddd90121ef4e0e64abcd22edea581d kind = "model" usage = "Offline multilingual OCR and scan orientation correction (cargo xtask models)" sha256 = "b5f2bfe2bdd9448429e3e82b51c789775d9b42f2403d082b00662eb77e401c5d" + +[[asset]] +path = "packaging/windows/locales/de.wxl" +title = "PdfCraft installer translation catalog (de)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "a2fb02e52e23b871339f92af3874fbd5c35fd7fb00e551c7b011e937d4f7d1c6" + +[[asset]] +path = "packaging/windows/locales/en.wxl" +title = "PdfCraft installer translation catalog (en)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "176e21d1bb8d473a78bb294ece6034e7d23644304ee04d53c65b14b380c895f6" + +[[asset]] +path = "packaging/windows/locales/es.wxl" +title = "PdfCraft installer translation catalog (es)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "151645e9eacac1844766594e331a4d86f33d3c070bcd2e8684f92193f0c40a21" + +[[asset]] +path = "packaging/windows/locales/fr.wxl" +title = "PdfCraft installer translation catalog (fr)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "1d246aff45ce25a07dd7cc12d14dd83e5ee95b1353e1a42ab8a2ca943b5c4b57" + +[[asset]] +path = "packaging/windows/locales/hu.wxl" +title = "PdfCraft installer translation catalog (hu)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "9b0a2ae2bc6d0cfb905a0fa543c92fa56582b227f71e54046a5b8108665dfe95" + +[[asset]] +path = "packaging/windows/locales/it.wxl" +title = "PdfCraft installer translation catalog (it)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "f38d7f6f452013d2a2da0c369451e73ba3fc18a2565c828b77ae45c14c3f5d14" + +[[asset]] +path = "packaging/windows/locales/nl.wxl" +title = "PdfCraft installer translation catalog (nl)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "f3c8e97ecb0e30a10d73b1884313b02e0848243fd6f1545ae0b8641fa6247b67" + +[[asset]] +path = "packaging/macos/resources/de.lproj/Localizable.strings" +title = "PdfCraft installer translation catalog (de)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "39fa7b20f8d9e08141c41927a3638785d963a682a75318a64bc4a36fea7b29ec" + +[[asset]] +path = "packaging/macos/resources/en.lproj/Localizable.strings" +title = "PdfCraft installer translation catalog (en)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "d4471cf4a88329a94f2dc87ebe50d5067c09505f7bdd966e41a0f388671eab1d" + +[[asset]] +path = "packaging/macos/resources/es.lproj/Localizable.strings" +title = "PdfCraft installer translation catalog (es)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "72e6e132c8b14875f7d1aac54a74d9958c2179ce131b02df3311d64efab2978c" + +[[asset]] +path = "packaging/macos/resources/fr.lproj/Localizable.strings" +title = "PdfCraft installer translation catalog (fr)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "9e0e98b3d8397a3322578be46c69c04e61a2ade02f73d6d4aeb7cab093d5129f" + +[[asset]] +path = "packaging/macos/resources/hu.lproj/Localizable.strings" +title = "PdfCraft installer translation catalog (hu)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "1bfb5836c5b0c2d7f148263c115a707951ed223a7b5924b763972a3f0226a4a2" + +[[asset]] +path = "packaging/macos/resources/it.lproj/Localizable.strings" +title = "PdfCraft installer translation catalog (it)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "d10653056aa636f37d6ca314c706880517cd0559c1d2a942e59070ca66f59da1" + +[[asset]] +path = "packaging/macos/resources/nl.lproj/Localizable.strings" +title = "PdfCraft installer translation catalog (nl)" +author = "PdfCraft contributors" +source = "Contributor-original: clean-room translations of PdfCraft installer labels; no proprietary localization resources" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "translation" +usage = "Native installer UI; English also supplies generated fallback locale catalogs" +sha256 = "a5d24537af99f5b607b014c395859508baa05b7f4b8dfec9da78e7c6539931ca" diff --git a/packaging/installer-languages.json b/packaging/installer-languages.json new file mode 100644 index 0000000000..c628a7ea16 --- /dev/null +++ b/packaging/installer-languages.json @@ -0,0 +1,44 @@ +{ + "en": { + "culture": "en-US", + "language": 1033, + "codepage": 1252, + "primary": 9 + }, + "de": { + "culture": "de-DE", + "language": 1031, + "codepage": 1252, + "primary": 7 + }, + "fr": { + "culture": "fr-FR", + "language": 1036, + "codepage": 1252, + "primary": 12 + }, + "es": { + "culture": "es-ES", + "language": 3082, + "codepage": 1252, + "primary": 10 + }, + "nl": { + "culture": "nl-NL", + "language": 1043, + "codepage": 1252, + "primary": 19 + }, + "it": { + "culture": "it-IT", + "language": 1040, + "codepage": 1252, + "primary": 16 + }, + "hu": { + "culture": "hu-HU", + "language": 1038, + "codepage": 1250, + "primary": 14 + } +} diff --git a/packaging/installer-languages.txt b/packaging/installer-languages.txt new file mode 100644 index 0000000000..1837a4f2ef --- /dev/null +++ b/packaging/installer-languages.txt @@ -0,0 +1,90 @@ +Installer languages +=================== + +PdfCraft-owned installation dialogs and optional Hot Folder descriptions are +translated into English, German, French, Spanish, Dutch, Italian and Hungarian. +The catalogs are contributor-original, licensed MIT OR Apache-2.0 and recorded +per file in ATTRIBUTION.toml. They contain no third-party artwork or fonts. +Native OS prompts (for example UAC and Apple's authentication sheet) are supplied +and localized by the operating system. + +Windows +------- +Use pdfcraft--windows--setup.exe for interactive installation. +This offline WiX Burn bundle embeds one MSI, including its six language MSTs, +and delegates all installation choices to that MSI's existing native dialogs. +Hot Folder remains optional and initially off. There is no extra language prompt, +installer network download, custom bootstrapper runtime, or OCR model duplication. + +Burn's UserUILanguageID selects the primary display language. Do not replace this +with UserLanguageID: that is the formatting locale and can differ from the UI. +All six sublanguage bits are accepted, including de-AT, fr-CA, es-MX, nl-BE and +it-CH. Unsupported primary languages leave the English base untouched. MSI +maintenance uses Windows Installer's cached installation transforms. + +The .msi is retained for administrative deployment; launching it directly uses +English unless the administrator supplies an embedded transform, for example: + + msiexec /i pdfcraft.msi TRANSFORMS=:de.mst INSTALLHOTFOLDER=1 + +Available transforms: de.mst, fr.mst, es.mst, nl.mst, it.mst, hu.mst. +Use TRANSFORMS=:hu.mst for Hungarian (database codepage 1250); the other catalogs +use 1252. The MSI's Template Summary lists all seven languages. Every localized +build shares a ProductCode, component GUIDs and payload; language transforms do +not change the service/account/ACL setup or optional feature state. + +package.ps1 invokes build-localized.ps1 and inspects all embedded transforms. +One CAB cache is reused and localized staging MSIs are removed after creating +their small transforms. Signing occurs after embedding the transforms. The Burn +engine is detached, signed, reattached, and the final bundle is signed as well. +Burn keeps the full release/SemVer version separately from the numeric MSI +ProductVersion so successive preview installers remain distinguishable. +Unsigned preview builds retain the existing explicit signing warning. + +macOS +----- +The PKG uses native Installer .lproj/Localizable.strings resources; the DMG has +no installer dialogs. Distribution choice descriptions are localization keys. +English is also the nonlocalized key text and fallback table. + +prepare-resources.sh obtains known ISO language codes from the build system's +Foundation NSLocale API and generates English fallback catalogs for other +languages. These are not additional translations: they prevent a primary +unsupported language from falling through to a translated secondary preference +(e.g. Japanese followed by German). Chinese script aliases are included too. +The seven actual catalogs stay separate and are never overwritten by fallbacks. +No installation-time scripts are added for localization. + +Validation +---------- + python packaging/test-installer-languages.py + pwsh packaging/windows/test-language-build.ps1 -Arch arm64 -Work NEW_SCRATCH + pwsh packaging/windows/test-language-build.ps1 -Arch x64 -Work NEW_SCRATCH + bash packaging/macos/test-localized-pkg.sh NEW_SCRATCH + +The Windows fixture builds real MSI/Setup containers with tiny inert payloads. +It applies each actual embedded transform to a disposable database, checks every +dialog/control string, language, codepage, package identity, service authoring +and feature default. It extracts the finished Setup, checks the embedded MSI +hash and PE architecture, and exhaustively verifies all 65,536 LANGIDs against +the compiled Burn language selectors. It never executes Setup or installs MSI. + +The macOS fixture invokes pkgbuild/productbuild, expands the real PKG, checks its +catalogs and default-off Hot Folder choice, and uses Apple's Foundation locale +resolver for regional and fallback cases. It never installs the package. +The installer-languages.yml workflow performs these checks on Windows (x64, +ARM64 and x86 authoring) and macOS without building the PDF/OCR application. +These are packaging tests; they do not claim visual inspection on every OS +display language or exercise certificate-backed signing without CI secrets. + +Primary technical references +---------------------------- +https://docs.firegiant.com/wix/tools/burn/builtin-variables/ +https://docs.firegiant.com/wix/schema/bal/wixinternaluibootstrapperapplication/ +https://docs.firegiant.com/wix/schema/wxs/setvariable/ +https://docs.firegiant.com/wix/tools/wixexe/ +https://docs.firegiant.com/wix/tools/signing/ +https://learn.microsoft.com/en-us/windows/win32/msi/embedded-transforms +https://learn.microsoft.com/en-us/windows/win32/msi/code-page-handling-of-parameter-strings +https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html +https://developer.apple.com/documentation/foundation/nslocale/isolanguagecodes diff --git a/packaging/macos/Distribution.xml.in b/packaging/macos/Distribution.xml.in index edc7f0cbb2..8e9fd696b7 100644 --- a/packaging/macos/Distribution.xml.in +++ b/packaging/macos/Distribution.xml.in @@ -4,7 +4,7 @@ - + &2; exit 1; fi +mkdir -p "$destination" +cp -R "$here/resources/"* "$destination/" +# Installer follows the system language preference list. English copies for the +# other macOS display languages prevent falling through to a supported *secondary* +# preference (e.g. Japanese, then German). These are fallbacks, not translations. +# Ask the build SDK for ISO language codes rather than maintaining a stale list +# of macOS languages. Only directory names are generated; every fallback is the +# same contributor-original English catalog. Script-specific Chinese aliases +# ensure regional preferences resolve through the matching native script. +fallbacks="$(osascript <<'APPLESCRIPT' +use framework "Foundation" +set languageCodes to current application's NSLocale's ISOLanguageCodes() as list +set AppleScript's text item delimiters to linefeed +return languageCodes as text +APPLESCRIPT +)" +while IFS= read -r language; do + case "$language" in en|de|fr|es|nl|it|hu) continue ;; esac + if [[ ! "$language" =~ ^[a-z]{2,3}$ ]]; then echo "Unexpected ISO language code: $language" >&2; exit 1; fi + mkdir -p "$destination/$language.lproj" + cp "$here/resources/en.lproj/Localizable.strings" "$destination/$language.lproj/Localizable.strings" +done <<< "$fallbacks" +for language in zh-Hans zh-Hant; do + mkdir -p "$destination/$language.lproj" + cp "$here/resources/en.lproj/Localizable.strings" "$destination/$language.lproj/Localizable.strings" +done +# A non-localized English table also covers missing/unknown locale resources. +cp "$here/resources/en.lproj/Localizable.strings" "$destination/Localizable.strings" +find "$destination" -name Localizable.strings -exec plutil -lint {} \; diff --git a/packaging/macos/resources/de.lproj/Localizable.strings b/packaging/macos/resources/de.lproj/Localizable.strings new file mode 100644 index 0000000000..1fce17c189 --- /dev/null +++ b/packaging/macos/resources/de.lproj/Localizable.strings @@ -0,0 +1,4 @@ +"PdfCraft" = "PdfCraft"; +"PdfCraft Hot Folder" = "PdfCraft Hot Folder"; +"Optional folder automation and a background service at system startup, running under your user account. Only tasks you configure and enable are processed." = "Optionale Ordnerautomatisierung und ein Hintergrunddienst beim Systemstart unter Ihrem Benutzerkonto. Nur von Ihnen eingerichtete und aktivierte Aufgaben werden verarbeitet."; +"Install the PdfCraft application for working with PDF documents." = "PdfCraft zum Bearbeiten von PDF-Dokumenten installieren."; diff --git a/packaging/macos/resources/en.lproj/Localizable.strings b/packaging/macos/resources/en.lproj/Localizable.strings new file mode 100644 index 0000000000..39b2024690 --- /dev/null +++ b/packaging/macos/resources/en.lproj/Localizable.strings @@ -0,0 +1,4 @@ +"PdfCraft" = "PdfCraft"; +"PdfCraft Hot Folder" = "PdfCraft Hot Folder"; +"Optional folder automation and a background service at system startup, running under your user account. Only tasks you configure and enable are processed." = "Optional folder automation and a background service at system startup, running under your user account. Only tasks you configure and enable are processed."; +"Install the PdfCraft application for working with PDF documents." = "Install the PdfCraft application for working with PDF documents."; diff --git a/packaging/macos/resources/es.lproj/Localizable.strings b/packaging/macos/resources/es.lproj/Localizable.strings new file mode 100644 index 0000000000..f1a10a9190 --- /dev/null +++ b/packaging/macos/resources/es.lproj/Localizable.strings @@ -0,0 +1,4 @@ +"PdfCraft" = "PdfCraft"; +"PdfCraft Hot Folder" = "PdfCraft Hot Folder"; +"Optional folder automation and a background service at system startup, running under your user account. Only tasks you configure and enable are processed." = "Automatización opcional de carpetas y servicio en segundo plano al iniciar el sistema, bajo su cuenta de usuario. Solo se procesan las tareas que configure y active."; +"Install the PdfCraft application for working with PDF documents." = "Instalar PdfCraft para trabajar con documentos PDF."; diff --git a/packaging/macos/resources/fr.lproj/Localizable.strings b/packaging/macos/resources/fr.lproj/Localizable.strings new file mode 100644 index 0000000000..94aade5c15 --- /dev/null +++ b/packaging/macos/resources/fr.lproj/Localizable.strings @@ -0,0 +1,4 @@ +"PdfCraft" = "PdfCraft"; +"PdfCraft Hot Folder" = "PdfCraft Hot Folder"; +"Optional folder automation and a background service at system startup, running under your user account. Only tasks you configure and enable are processed." = "Automatisation facultative des dossiers et service en arrière-plan au démarrage du système, sous votre compte utilisateur. Seules les tâches que vous configurez et activez sont traitées."; +"Install the PdfCraft application for working with PDF documents." = "Installer PdfCraft pour travailler sur des documents PDF."; diff --git a/packaging/macos/resources/hu.lproj/Localizable.strings b/packaging/macos/resources/hu.lproj/Localizable.strings new file mode 100644 index 0000000000..a3039e62ec --- /dev/null +++ b/packaging/macos/resources/hu.lproj/Localizable.strings @@ -0,0 +1,4 @@ +"PdfCraft" = "PdfCraft"; +"PdfCraft Hot Folder" = "PdfCraft Hot Folder"; +"Optional folder automation and a background service at system startup, running under your user account. Only tasks you configure and enable are processed." = "Választható mappaautomatizálás és rendszerindításkor induló háttérszolgáltatás az Ön felhasználói fiókjával. Csak a beállított és engedélyezett feladatokat dolgozza fel."; +"Install the PdfCraft application for working with PDF documents." = "A PdfCraft telepítése PDF-dokumentumok kezeléséhez."; diff --git a/packaging/macos/resources/it.lproj/Localizable.strings b/packaging/macos/resources/it.lproj/Localizable.strings new file mode 100644 index 0000000000..5c36560946 --- /dev/null +++ b/packaging/macos/resources/it.lproj/Localizable.strings @@ -0,0 +1,4 @@ +"PdfCraft" = "PdfCraft"; +"PdfCraft Hot Folder" = "PdfCraft Hot Folder"; +"Optional folder automation and a background service at system startup, running under your user account. Only tasks you configure and enable are processed." = "Automazione facoltativa delle cartelle e servizio in background all'avvio del sistema, con il tuo account utente. Vengono elaborate solo le attività configurate e attivate."; +"Install the PdfCraft application for working with PDF documents." = "Installa PdfCraft per lavorare con i documenti PDF."; diff --git a/packaging/macos/resources/nl.lproj/Localizable.strings b/packaging/macos/resources/nl.lproj/Localizable.strings new file mode 100644 index 0000000000..d01ffc7961 --- /dev/null +++ b/packaging/macos/resources/nl.lproj/Localizable.strings @@ -0,0 +1,4 @@ +"PdfCraft" = "PdfCraft"; +"PdfCraft Hot Folder" = "PdfCraft Hot Folder"; +"Optional folder automation and a background service at system startup, running under your user account. Only tasks you configure and enable are processed." = "Optionele mapautomatisering en een achtergrondservice bij het opstarten, onder uw gebruikersaccount. Alleen taken die u instelt en inschakelt worden verwerkt."; +"Install the PdfCraft application for working with PDF documents." = "PdfCraft installeren om met PDF-documenten te werken."; diff --git a/packaging/macos/test-localization.applescript b/packaging/macos/test-localization.applescript new file mode 100644 index 0000000000..c6ff770467 --- /dev/null +++ b/packaging/macos/test-localization.applescript @@ -0,0 +1,32 @@ +-- Tests Apple's native locale resolution against the resources expanded from a PKG. +-- This only reads resources. It never starts Installer or changes language preferences. +use framework "Foundation" +use scripting additions + +on run arguments + set resourcePath to item 1 of arguments + set fileManager to current application's NSFileManager's defaultManager() + set names to fileManager's contentsOfDirectoryAtPath:resourcePath |error|:(missing value) + set available to {} + repeat with resourceName in (names as list) + if resourceName ends with ".lproj" then set end of available to text 1 thru -7 of resourceName + end repeat + set cases to {{"en-AU", "en"}, {"de-AT", "de"}, {"fr-CA", "fr"}, {"es-MX", "es"}, {"nl-BE", "nl"}, {"it-CH", "it"}, {"hu-HU", "hu"}, {"ja-JP", "ja"}, {"ru-RU", "ru"}, {"pt-BR", "pt"}, {"zh-TW", "zh-Hant"}} + repeat with testCase in cases + set preferred to current application's NSBundle's preferredLocalizationsFromArray:available forPreferences:{item 1 of testCase, "de", "en"} + set chosen to item 1 of (preferred as list) + if chosen is not item 2 of testCase then error "Unexpected locale for " & item 1 of testCase & ": " & chosen + set tablePath to resourcePath & "/" & chosen & ".lproj/Localizable.strings" + set table to current application's NSDictionary's dictionaryWithContentsOfFile:tablePath + if table is missing value then error "Unreadable localization: " & tablePath + set key to "Install the PdfCraft application for working with PDF documents." + set value to table's objectForKey:key + if value is missing value then error "Missing PdfCraft choice description: " & chosen + if chosen is in {"en", "ja", "ru", "pt", "zh-Hant"} then + if (value as text) is not key then error "Non-English unsupported-language fallback: " & chosen + else + if (value as text) is key then error "Missing translated choice description: " & chosen + end if + end repeat + return "ok: native locale resolution, regional variants, primary-language English fallback and PKG strings" +end run diff --git a/packaging/macos/test-localized-pkg.sh b/packaging/macos/test-localized-pkg.sh new file mode 100644 index 0000000000..6d3ac4c50d --- /dev/null +++ b/packaging/macos/test-localized-pkg.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Tiny real productbuild fixture; no installation, daemon, signing identity or app build. +set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +work="${1:?usage: test-localized-pkg.sh NEW_SCRATCH_DIRECTORY}" +if [ -e "$work" ]; then echo "test directory already exists: $work" >&2; exit 1; fi +mkdir -p "$work/components" "$work/empty" +for component in PdfCraft HotFolder; do + identifier=ai.storyteller.pdfcraft.pkg + [ "$component" = HotFolder ] && identifier=ai.storyteller.pdfcraft.hot-folder.pkg + pkgbuild --root "$work/empty" --identifier "$identifier" --version 0.6.0 "$work/components/$component-component.pkg" +done +sed 's/@SHORT_VERSION@/0.6.0/g' "$here/Distribution.xml.in" > "$work/Distribution.xml" +bash "$here/prepare-resources.sh" "$work/resources" +productbuild --distribution "$work/Distribution.xml" --package-path "$work/components" --resources "$work/resources" "$work/localized.pkg" +pkgutil --expand "$work/localized.pkg" "$work/expanded" +for language in en de fr es nl it hu; do + cmp "$here/resources/$language.lproj/Localizable.strings" "$work/expanded/Resources/$language.lproj/Localizable.strings" +done +osascript "$here/test-localization.applescript" "$work/expanded/Resources" +# The default-off service choice must survive productbuild, not merely source parsing. +installer -showChoicesXML -pkg "$work/localized.pkg" -target / > "$work/choices.plist" +plutil -lint "$work/choices.plist" +python3 - "$work/choices.plist" <<'PY' +import plistlib, sys +with open(sys.argv[1], 'rb') as f: + choices = plistlib.load(f) +def walk(items): + for item in items: + yield item + yield from walk(item.get('childItems', [])) +hot = [x for x in walk(choices) if x.get('choiceIdentifier') == 'hot-folder'] +assert len(hot) == 1 and not hot[0]['choiceIsSelected'], hot +PY diff --git a/packaging/service-tests/macos-pkg.sh b/packaging/service-tests/macos-pkg.sh index 6ea2987221..cf48d03505 100755 --- a/packaging/service-tests/macos-pkg.sh +++ b/packaging/service-tests/macos-pkg.sh @@ -20,7 +20,8 @@ pkgbuild --component "$main" --install-location /Applications --identifier ai.st pkgbuild --component "$hot" --install-location /Applications --identifier ai.storyteller.pdfcraft.hot-folder.pkg \ --version 0.6.0 --scripts "$work/scripts" "$work/components/HotFolder-component.pkg" sed 's/@SHORT_VERSION@/0.6.0/g' packaging/macos/Distribution.xml.in > "$work/Distribution.xml" -productbuild --distribution "$work/Distribution.xml" --package-path "$work/components" "$work/PdfCraft.pkg" +bash packaging/macos/prepare-resources.sh "$work/resources" +productbuild --resources "$work/resources" --distribution "$work/Distribution.xml" --package-path "$work/components" "$work/PdfCraft.pkg" sudo installer -pkg "$work/PdfCraft.pkg" -target / test -d /Applications/PdfCraft.app test ! -d '/Applications/PdfCraft Hot Folder.app' diff --git a/packaging/service-tests/windows-msi.ps1 b/packaging/service-tests/windows-msi.ps1 index 5db8ca3163..33f09c7d9a 100644 --- a/packaging/service-tests/windows-msi.ps1 +++ b/packaging/service-tests/windows-msi.ps1 @@ -15,7 +15,7 @@ $upgrade = Join-Path $stage 'service-test-upgrade.msi' $installedMsi = $msi & wix extension add -g WixToolset.Util.wixext/5.0.2 if ($LASTEXITCODE -ne 0) { throw 'WiX extension installation failed' } -& wix build packaging/windows/pdfcraft.wxs packaging/windows/installer-ui.wxs -arch x64 -ext WixToolset.Util.wixext -d Version=0.6.0 -d "BinDir=$stage" -d "ModelsDir=$stage/models" -d 'IconPath=assets/app-icon/pdfcraft.ico' -o $msi +& wix build packaging/windows/pdfcraft.wxs packaging/windows/installer-ui.wxs -loc packaging/windows/locales/en.wxl -arch x64 -ext WixToolset.Util.wixext -d Version=0.6.0 -d "BinDir=$stage" -d "ModelsDir=$stage/models" -d 'IconPath=assets/app-icon/pdfcraft.ico' -o $msi if ($LASTEXITCODE -ne 0) { throw 'WiX compilation failed' } & pwsh -NoProfile -File packaging/windows/test-msi.ps1 $msi if ($LASTEXITCODE -ne 0) { throw 'Compiled MSI validation failed' } @@ -24,7 +24,7 @@ if ($LASTEXITCODE -ne 0) { throw 'Compiled MSI validation failed' } $probeStream = [IO.File]::Open("$stage/pdfcraft-hot-folder.exe", [IO.FileMode]::Append) try { $probeStream.Write([Text.Encoding]::ASCII.GetBytes('synthetic upgrade payload')) } finally { $probeStream.Dispose() } $upgradeHash = (Get-FileHash -LiteralPath "$stage/pdfcraft-hot-folder.exe" -Algorithm SHA256).Hash -& wix build packaging/windows/pdfcraft.wxs packaging/windows/installer-ui.wxs -arch x64 -ext WixToolset.Util.wixext -d Version=0.6.1 -d "BinDir=$stage" -d "ModelsDir=$stage/models" -d 'IconPath=assets/app-icon/pdfcraft.ico' -o $upgrade +& wix build packaging/windows/pdfcraft.wxs packaging/windows/installer-ui.wxs -loc packaging/windows/locales/en.wxl -arch x64 -ext WixToolset.Util.wixext -d Version=0.6.1 -d "BinDir=$stage" -d "ModelsDir=$stage/models" -d 'IconPath=assets/app-icon/pdfcraft.ico' -o $upgrade if ($LASTEXITCODE -ne 0) { throw 'Upgrade MSI compilation failed' } function Invoke-Msi([string] $Options, [string] $Name) { $log = Join-Path $stage "$Name.log" diff --git a/packaging/test-installer-languages.py b/packaging/test-installer-languages.py new file mode 100644 index 0000000000..0d495c7e73 --- /dev/null +++ b/packaging/test-installer-languages.py @@ -0,0 +1,67 @@ +"""Source/catalog parity. Compiled MSI and native PKG checks run separately.""" +import json +import pathlib +import re +import xml.etree.ElementTree as ET + +root = pathlib.Path(__file__).resolve().parent +languages = json.loads((root / 'installer-languages.json').read_text(encoding='utf-8')) +assert set(languages) == {'en', 'de', 'fr', 'es', 'nl', 'it', 'hu'} +assert {p.stem for p in (root / 'windows' / 'locales').glob('*.wxl')} == set(languages) +assert {p.stem for p in (root / 'macos' / 'resources').glob('*.lproj')} == set(languages) +catalogs = {} +for language, metadata in languages.items(): + xml = ET.parse(root / 'windows' / 'locales' / (language + '.wxl')).getroot() + strings = {x.attrib['Id']: x.attrib['Value'] for x in xml} + assert len(strings) == len(xml), (language, 'duplicate strings') + assert xml.attrib['Culture'] == metadata['culture'] + assert int(xml.attrib['Codepage']) == metadata['codepage'] + assert int(strings['Language']) == metadata['language'] + assert int(strings['Codepage']) == metadata['codepage'] + for value in strings.values(): + assert value.strip(), (language, 'empty translation') + assert value.encode('cp' + str(metadata['codepage'])).decode('cp' + str(metadata['codepage'])) == value + catalogs[language] = strings +reference = catalogs['en'] +for language, strings in catalogs.items(): + assert strings.keys() == reference.keys(), language + for key, value in strings.items(): + assert re.findall(r'\[[^]]+\]', value) == re.findall(r'\[[^]]+\]', reference[key]), (language, key, 'placeholders') + +used = set() +for file in ['pdfcraft.wxs', 'installer-ui.wxs']: + xml = ET.parse(root / 'windows' / file).getroot() + for element in xml.iter(): + for name, value in element.attrib.items(): + used.update(re.findall(r'!\(loc\.([A-Za-z]+)\)', value)) + if name == 'Id': + assert '!(loc.' not in value, (file, 'localized identifier') + if element.tag.endswith('}Control') and 'Text' in element.attrib: + assert '!(loc.' in element.attrib['Text'], element.attrib + if element.tag.endswith('}Dialog'): + assert '!(loc.' in element.attrib['Title'], element.attrib + width, height = int(element.attrib['Width']), int(element.attrib['Height']) + for control in element: + if not control.tag.endswith('}Control'): + continue + a = control.attrib + assert int(a['X']) + int(a['Width']) <= width, a + assert int(a['Y']) + int(a['Height']) <= height, a +assert used == set(reference), (used - set(reference), set(reference) - used) + +mac_reference = None +for language in languages: + source = (root / 'macos' / 'resources' / (language + '.lproj') / 'Localizable.strings').read_text(encoding='utf-8') + table = {} + for line in source.splitlines(): + if not line.strip() or line.startswith('//'): + continue + match = re.fullmatch(r'("(?:[^"\\]|\\.)*") = ("(?:[^"\\]|\\.)*");', line) + assert match, (language, line) + key, value = [json.loads(x) for x in match.groups()] + assert key not in table and value.strip(), (language, key) + table[key] = value + if mac_reference is None: + mac_reference = table.keys() + assert table.keys() == mac_reference, language +print('ok: seven complete catalogs, placeholders, codepages, native UI bounds, macOS strings') diff --git a/packaging/windows/build-localized.ps1 b/packaging/windows/build-localized.ps1 new file mode 100644 index 0000000000..b9d682e54a --- /dev/null +++ b/packaging/windows/build-localized.ps1 @@ -0,0 +1,73 @@ +<# Build the offline localized MSI/Setup pair from an already validated payload. + Does not execute an installer. Temporary localized MSIs share a CAB cache and + ProductCode; only the English base plus small embedded transforms is published. #> +param( + [Parameter(Mandatory)] [ValidateSet('x64', 'x86', 'arm64')] [string] $Arch, + [Parameter(Mandatory)] [string] $Version, + [string] $BundleVersion = $Version, + [Parameter(Mandatory)] [string] $BinDir, + [Parameter(Mandatory)] [string] $ModelsDir, + [Parameter(Mandatory)] [string] $IconPath, + [Parameter(Mandatory)] [string] $Msi, + [Parameter(Mandatory)] [string] $Setup, + [Parameter(Mandatory)] [string] $Work +) +$ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'installer-languages.ps1') +function Invoke-Wix([string[]] $Arguments) { + & wix @Arguments + if ($LASTEXITCODE -ne 0) { throw "WiX $($Arguments[0]) failed ($LASTEXITCODE)" } +} +$Work = [IO.Path]::GetFullPath($Work) +New-Item -ItemType Directory -Force -Path $Work | Out-Null +$Msi = [IO.Path]::GetFullPath($Msi) +$Setup = [IO.Path]::GetFullPath($Setup) +$productCode = [Guid]::NewGuid().ToString('B').ToUpperInvariant() +Invoke-Wix @('extension', 'add', '-g', 'WixToolset.Util.wixext/5.0.2') +Invoke-Wix @('extension', 'add', '-g', 'WixToolset.BootstrapperApplications.wixext/5.0.2') +$languageFiles = @{} +foreach ($language in @('en', 'de', 'fr', 'es', 'nl', 'it', 'hu')) { + $path = Join-Path $Work "$language.msi" + Invoke-Wix @('build', (Join-Path $PSScriptRoot 'pdfcraft.wxs'), (Join-Path $PSScriptRoot 'installer-ui.wxs'), + '-arch', $Arch, '-ext', 'WixToolset.Util.wixext', '-loc', (Join-Path $PSScriptRoot "locales/$language.wxl"), + '-cc', (Join-Path $Work 'cab-cache'), '-d', "ProductCode=$productCode", '-d', "Version=$Version", + '-d', "BinDir=$BinDir", '-d', "ModelsDir=$ModelsDir", '-d', "IconPath=$IconPath", '-o', $path) + $languageFiles[$language] = $path + if ($language -ne 'en') { + Invoke-Wix @('msi', 'transform', '-t', 'language', $languageFiles.en, $path, '-o', (Join-Path $Work "$language.mst")) + # OCR payloads are large. Keep only the base plus the current localization, + # not seven copies of the same models on a size-limited release runner. + Remove-Item -LiteralPath $path + } + Remove-Item -LiteralPath ([IO.Path]::ChangeExtension($path, '.wixpdb')) -ErrorAction SilentlyContinue +} +Copy-Item -LiteralPath $languageFiles.en -Destination $Msi -Force +# Add transformations before signing: signature covers every language and all CABs. +& (Get-Process -Id $PID).Path -NoProfile -File (Join-Path $PSScriptRoot 'embed-transforms.ps1') -Msi $Msi -Work $Work +if ($LASTEXITCODE -ne 0) { throw 'Embedding MSI translations failed' } +# A child process releases all Windows Installer database handles before signing. +& (Get-Process -Id $PID).Path -NoProfile -File (Join-Path $PSScriptRoot 'test-localized.ps1') -Msi $Msi -Work $Work +if ($LASTEXITCODE -ne 0) { throw 'Localized MSI validation failed' } +& (Join-Path $PSScriptRoot 'sign.ps1') $Msi + +$properties = @('') +foreach ($language in @('de', 'fr', 'es', 'nl', 'it', 'hu')) { + $condition = Get-InstallerLanguageCondition $language + $properties += " " +} +$properties += '' +$include = Join-Path $Work 'language-properties.wxi' +[IO.File]::WriteAllLines($include, $properties, [Text.UTF8Encoding]::new($false)) +$unsigned = Join-Path $Work 'setup-unsigned.exe' +Invoke-Wix @('build', (Join-Path $PSScriptRoot 'setup.wxs'), '-arch', $Arch, + '-ext', 'WixToolset.BootstrapperApplications.wixext', '-d', "Version=$BundleVersion", '-d', "MsiPath=$Msi", + '-d', "IconPath=$IconPath", '-d', "LanguageProperties=$include", '-o', $unsigned) +# Burn caches its engine for maintenance. Sign it as well as the outer container. +$engine = Join-Path $Work 'setup-engine.exe' +Invoke-Wix @('burn', 'detach', $unsigned, '-engine', $engine) +& (Join-Path $PSScriptRoot 'sign.ps1') $engine +Invoke-Wix @('burn', 'reattach', $unsigned, '-engine', $engine, '-o', $Setup) +& (Join-Path $PSScriptRoot 'sign.ps1') $Setup +& (Get-Process -Id $PID).Path -NoProfile -File (Join-Path $PSScriptRoot 'test-setup.ps1') ` + -Setup $Setup -Msi $Msi -Work (Join-Path $Work 'inspect-setup') -Arch $Arch +if ($LASTEXITCODE -ne 0) { throw 'Offline Setup validation failed' } diff --git a/packaging/windows/embed-transforms.ps1 b/packaging/windows/embed-transforms.ps1 new file mode 100644 index 0000000000..aff3842b0b --- /dev/null +++ b/packaging/windows/embed-transforms.ps1 @@ -0,0 +1,27 @@ +# Update only a build artifact; a separate process releases all MSI stream handles. +param([Parameter(Mandatory)] [string] $Msi, [Parameter(Mandatory)] [string] $Work) +$ErrorActionPreference = 'Stop' +$installer = New-Object -ComObject WindowsInstaller.Installer +$database = $null +try { + $database = $installer.OpenDatabase($Msi, 1) + foreach ($language in @('de', 'fr', 'es', 'nl', 'it', 'hu')) { + $view = $database.OpenView('INSERT INTO `_Storages` (`Name`, `Data`) VALUES (?, ?)') + try { + $record = $installer.CreateRecord(2) + $record.StringData(1) = "$language.mst" + [void] $record.SetStream(2, (Join-Path $Work "$language.mst")) + [void] $view.Execute($record) + } finally { [void] $view.Close() } + } + # The multilingual package advertises all embedded languages in its Template + # Summary; ProductLanguage is still the English base until a transform is applied. + $summary = $database.SummaryInformation(1) + $platform = ($summary.Property(7) -split ';')[0] + $summary.Property(7) = "$platform;1033,1031,1036,3082,1043,1040,1038" + [void] $summary.Persist() + [void] $database.Commit() +} finally { + if ($database) { [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($database) } + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($installer) +} diff --git a/packaging/windows/installer-languages.ps1 b/packaging/windows/installer-languages.ps1 new file mode 100644 index 0000000000..264667338c --- /dev/null +++ b/packaging/windows/installer-languages.ps1 @@ -0,0 +1,27 @@ +# Shared build-time language metadata. Selection uses UI language, never formatting locale. +$InstallerLanguages = Get-Content (Join-Path $PSScriptRoot '../installer-languages.json') -Raw -Encoding utf8 | ConvertFrom-Json -AsHashtable + +function Get-InstallerLanguage([int] $LanguageId) { + $primary = $LanguageId -band 0x3ff + foreach ($entry in $InstallerLanguages.GetEnumerator()) { + if ($entry.Value.primary -eq $primary) { return $entry.Key } + } + return 'en' +} + +function Get-InstallerLanguageCondition([string] $Language) { + # A LANGID has ten primary-language bits and six sublanguage bits. Burn conditions + # have no arithmetic/bitwise operator, so enumerate those 64 possible LANGIDs. + # This also covers regional variants not explicitly listed by .NET CultureInfo. + $primary = $InstallerLanguages[$Language].primary + if (-not $primary) { throw "Unknown installer language: $Language" } + return ((0..63 | ForEach-Object { "UserUILanguageID = $($primary + 1024 * $_)" }) -join ' OR ') +} + +function Get-InstallerStrings([string] $Language) { + if (-not $InstallerLanguages.ContainsKey($Language)) { throw "Unknown installer language: $Language" } + [xml] $xml = Get-Content (Join-Path $PSScriptRoot "locales/$Language.wxl") -Raw -Encoding utf8 + $strings = @{} + foreach ($item in $xml.WixLocalization.String) { $strings[$item.Id] = $item.Value } + return $strings +} diff --git a/packaging/windows/installer-ui.wxs b/packaging/windows/installer-ui.wxs index 63b4436403..fb1cefceee 100644 --- a/packaging/windows/installer-ui.wxs +++ b/packaging/windows/installer-ui.wxs @@ -8,101 +8,101 @@ - - - - - - - + + + + + + + - + - - - - + + + + - + - + - + - + - - + + - - + + - + - - - + + + - + - - - - + + + + - - - - + + + + - - - - + + + + @@ -110,16 +110,16 @@ - - - - - + + + + + - + diff --git a/packaging/windows/locales/de.wxl b/packaging/windows/locales/de.wxl new file mode 100644 index 0000000000..d0f8624ca3 --- /dev/null +++ b/packaging/windows/locales/de.wxl @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/windows/locales/en.wxl b/packaging/windows/locales/en.wxl new file mode 100644 index 0000000000..9fe0bba026 --- /dev/null +++ b/packaging/windows/locales/en.wxl @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/windows/locales/es.wxl b/packaging/windows/locales/es.wxl new file mode 100644 index 0000000000..49971a2091 --- /dev/null +++ b/packaging/windows/locales/es.wxl @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/windows/locales/fr.wxl b/packaging/windows/locales/fr.wxl new file mode 100644 index 0000000000..b9e6bf35ea --- /dev/null +++ b/packaging/windows/locales/fr.wxl @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/windows/locales/hu.wxl b/packaging/windows/locales/hu.wxl new file mode 100644 index 0000000000..7647ea7007 --- /dev/null +++ b/packaging/windows/locales/hu.wxl @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/windows/locales/it.wxl b/packaging/windows/locales/it.wxl new file mode 100644 index 0000000000..0369ea4f71 --- /dev/null +++ b/packaging/windows/locales/it.wxl @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/windows/locales/nl.wxl b/packaging/windows/locales/nl.wxl new file mode 100644 index 0000000000..1c1a159013 --- /dev/null +++ b/packaging/windows/locales/nl.wxl @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/windows/package.ps1 b/packaging/windows/package.ps1 index bf32133b21..4b58afe0ff 100644 --- a/packaging/windows/package.ps1 +++ b/packaging/windows/package.ps1 @@ -4,6 +4,7 @@ .DESCRIPTION Produces, in $env:DIST (default: dist/release): + pdfcraft--windows--setup.exe offline installer, automatic UI language pdfcraft--windows-.msi per-machine installer (WiX v5) pdfcraft--windows--portable.zip pdfcraft.exe + pdfcraft-cli.exe + models\ + portable.txt @@ -128,19 +129,12 @@ if (-not (Test-Path (Join-Path $Models 'ATTRIBUTION.txt')) -or -not (Get-ChildIt # ---- MSI --------------------------------------------------------------------------------------- $Msi = Join-Path $Dist "pdfcraft-$Version-windows-$Arch.msi" -Invoke-Native 'WiX service configuration extension' { wix extension add -g WixToolset.Util.wixext/5.0.2 } -Invoke-Native 'wix build' { - wix build (Join-Path $PSScriptRoot 'pdfcraft.wxs') -arch $Arch -ext WixToolset.Util.wixext ` - (Join-Path $PSScriptRoot 'installer-ui.wxs') ` - -d "Version=$MsiVersion" -d "BinDir=$Stage" -d "ModelsDir=$Models" -d "IconPath=$(Join-Path $Root 'assets\app-icon\pdfcraft.ico')" ` - -o $Msi -} -# Inspect the built MSI, not just the XML, before signing/publishing it. In a child process, so -# its Windows Installer database handle is gone before signtool opens the MSI. +$Setup = Join-Path $Dist "pdfcraft-$Version-windows-$Arch-setup.exe" +& (Join-Path $PSScriptRoot 'build-localized.ps1') -Arch $Arch -Version $MsiVersion -BundleVersion $Version ` + -BinDir $Stage -ModelsDir $Models -IconPath (Join-Path $Root 'assets/app-icon/pdfcraft.ico') ` + -Msi $Msi -Setup $Setup -Work (Join-Path $Stage 'installer-languages') +# Inspect service, scope, shortcut and model authoring after language embedding. Invoke-Native 'test-msi' { & (Get-Process -Id $PID).Path -NoProfile -File (Join-Path $PSScriptRoot 'test-msi.ps1') $Msi } -# wix writes its debug symbols (.wixpdb) next to the MSI; keep them out of the release assets. -Remove-Item -Force -ErrorAction SilentlyContinue ([IO.Path]::ChangeExtension($Msi, '.wixpdb')) -& (Join-Path $PSScriptRoot 'sign.ps1') $Msi # ---- portable zip ------------------------------------------------------------------------------ $Portable = Join-Path $TargetDir "windows-package\pdfcraft-$Version-windows-$Arch-portable" @@ -178,4 +172,4 @@ if ($Arch -ne 'arm64' -or $HostArch -eq 'arm64') { } else { Write-Output "skipping pdfcraft-cli --version: an $Arch build doesn't run on this $HostArch machine" } -Get-Item $Msi, $Zip | Format-Table Name, Length +Get-Item $Setup, $Msi, $Zip | Format-Table Name, Length diff --git a/packaging/windows/pdfcraft.wxs b/packaging/windows/pdfcraft.wxs index 77f0b20b4e..80b94611d1 100644 --- a/packaging/windows/pdfcraft.wxs +++ b/packaging/windows/pdfcraft.wxs @@ -2,7 +2,7 @@ + + + - + Language="!(loc.Language)" Codepage="!(loc.Codepage)"> + + Message="!(loc.PackageScopeError)" /> + DowngradeErrorMessage="!(loc.DowngradeError)" /> @@ -84,7 +88,7 @@ + Description="!(loc.HotFolderFeatureDescription)" Level="2" AllowAdvertise="no"> @@ -95,7 +99,7 @@ @@ -123,7 +127,7 @@ @@ -142,7 +146,7 @@ .pdf's own default value is left alone, and the user chooses. Without this, the only PdfCraft entry to pick was the advertised Start Menu shortcut, which Windows can't run as a file handler ("This app can't run on your PC"). --> - + @@ -153,40 +157,40 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -197,14 +201,14 @@ (#143). Each has its own component with an HKLM key path, so Windows Installer removes it on uninstall; in this per-machine package both folders are the all-users ones. --> - - diff --git a/packaging/windows/setup.wxs b/packaging/windows/setup.wxs new file mode 100644 index 0000000000..8968e5c5a3 --- /dev/null +++ b/packaging/windows/setup.wxs @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + diff --git a/packaging/windows/test-language-build.ps1 b/packaging/windows/test-language-build.ps1 new file mode 100644 index 0000000000..ffff9e39e4 --- /dev/null +++ b/packaging/windows/test-language-build.ps1 @@ -0,0 +1,19 @@ +# Tiny compile-only payload. Nothing from this fixture is installed or published. +param([ValidateSet('x64', 'x86', 'arm64')] [string] $Arch = 'x64', + [Parameter(Mandatory)] [string] $Work) +$ErrorActionPreference = 'Stop' +if (Test-Path -LiteralPath $Work) { throw "Fixture directory must be new: $Work" } +$Work = [IO.Path]::GetFullPath($Work) +New-Item -ItemType Directory -Force -Path "$Work/bin/models", "$Work/out", "$Work/build" | Out-Null +foreach ($name in 'pdfcraft.exe', 'pdfcraft-cli.exe', 'pdfcraft-hot-folder.exe', 'LICENSE-MIT', 'LICENSE-APACHE', 'NOTICE', 'LICENSE-document.txt') { + [IO.File]::WriteAllText("$Work/bin/$name", 'Synthetic localization fixture. Not executable.', [Text.UTF8Encoding]::new($false)) +} +foreach ($name in 'synthetic-detection.rten', 'synthetic-recognition.rten') { + [IO.File]::WriteAllText("$Work/bin/models/$name", 'Synthetic model placeholder.', [Text.UTF8Encoding]::new($false)) +} +$root = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +& (Join-Path $PSScriptRoot 'build-localized.ps1') -Arch $Arch -Version 0.6.0 -BundleVersion 0.6.0-languages.1 ` + -BinDir "$Work/bin" -ModelsDir "$Work/bin/models" -IconPath "$root/assets/app-icon/pdfcraft.ico" ` + -Msi "$Work/out/PdfCraft.msi" -Setup "$Work/out/Setup.exe" -Work "$Work/build" +& (Get-Process -Id $PID).Path -NoProfile -File (Join-Path $PSScriptRoot 'test-msi.ps1') "$Work/out/PdfCraft.msi" +if ($LASTEXITCODE -ne 0) { throw 'Native MSI authoring checks failed' } diff --git a/packaging/windows/test-localized.ps1 b/packaging/windows/test-localized.ps1 new file mode 100644 index 0000000000..d21dd5f568 --- /dev/null +++ b/packaging/windows/test-localized.ps1 @@ -0,0 +1,63 @@ +# Inspect all compiled native dialogs and apply each actual embedded MST to a +# disposable MSI database. No product is installed and no custom action is run. +param([Parameter(Mandatory)] [string] $Msi, [Parameter(Mandatory)] [string] $Work) +$ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'installer-languages.ps1') +$installer = New-Object -ComObject WindowsInstaller.Installer +function Read-Value($Database, [string] $Sql) { + $view = $Database.OpenView($Sql) + $record = $null + try { + [void] $view.Execute() + $record = $view.Fetch() + if (-not $record) { throw "Missing row: $Sql" } + return $record.StringData(1) + } finally { + if ($record) { [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($record) } + [void] $view.Close() + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($view) + } +} +function Equal($Actual, $Expected, [string] $What) { + if ($Actual -cne $Expected) { throw "$What`: expected '$Expected', got '$Actual'" } +} +try { + [xml] $ui = Get-Content (Join-Path $PSScriptRoot 'installer-ui.wxs') -Raw -Encoding utf8 + $productCode = $null + foreach ($language in @('en', 'de', 'fr', 'es', 'nl', 'it', 'hu')) { + $copy = Join-Path $Work "verify-$language.msi" + Copy-Item -LiteralPath $Msi -Destination $copy -Force + $database = $installer.OpenDatabase($copy, 1) + try { + if ($language -ne 'en') { + # This uses the storage in the final MSI, not the loose build-time .mst. + [void] $database.ApplyTransform(":$language.mst", 32) + } + $strings = Get-InstallerStrings $language + $codepageFile = "codepage-$language.idt" + [void] $database.Export('_ForceCodepage', [IO.Path]::GetFullPath($Work), $codepageFile) + $codepage = Get-Content (Join-Path $Work $codepageFile) -Raw + if ($codepage -notmatch '(?m)^(\d+)\t_ForceCodepage') { throw 'Missing MSI database codepage' } + Equal $Matches[1] ([string] $InstallerLanguages[$language].codepage) "$language database codepage" + $summary = $database.SummaryInformation(0) + Equal (($summary.Property(7) -split ';')[1]) '1033,1031,1036,3082,1043,1040,1038' 'Multilingual package summary' + [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($summary) + Equal (Read-Value $database 'SELECT `Value` FROM `Property` WHERE `Property` = ''ProductLanguage''') ([string] $InstallerLanguages[$language].language) "$language ProductLanguage" + $code = Read-Value $database 'SELECT `Value` FROM `Property` WHERE `Property` = ''ProductCode''' + if ($productCode) { Equal $code $productCode "$language ProductCode" } else { $productCode = $code } + foreach ($dialog in $ui.Wix.Fragment.UI.Dialog) { + $expectedTitle = [regex]::Replace($dialog.Title, '!\(loc\.([A-Za-z]+)\)', { param($m) $strings[$m.Groups[1].Value] }) + Equal (Read-Value $database ('SELECT `Title` FROM `Dialog` WHERE `Dialog` = ''' + $dialog.Id + '''')) $expectedTitle "$language $($dialog.Id) title" + foreach ($control in $dialog.Control) { + if (-not $control.Text) { continue } + $expected = [regex]::Replace($control.Text, '!\(loc\.([A-Za-z]+)\)', { param($m) $strings[$m.Groups[1].Value] }) + $actual = Read-Value $database ('SELECT `Text` FROM `Control` WHERE `Dialog_` = ''' + $dialog.Id + ''' AND `Control` = ''' + $control.Id + '''') + Equal $actual $expected "$language $($dialog.Id).$($control.Id)" + } + } + Equal (Read-Value $database 'SELECT `Level` FROM `Feature` WHERE `Feature` = ''HotFolder''') '2' "$language Hot Folder remains opt-in" + Write-Output "ok $language`: embedded transform, ProductCode, language ID, all native dialogs and opt-in feature" + } finally { [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($database) } + Remove-Item -LiteralPath $copy + } +} finally { [void] [Runtime.InteropServices.Marshal]::FinalReleaseComObject($installer) } diff --git a/packaging/windows/test-setup.ps1 b/packaging/windows/test-setup.ps1 new file mode 100644 index 0000000000..2ab2f8c4b4 --- /dev/null +++ b/packaging/windows/test-setup.ps1 @@ -0,0 +1,48 @@ +# Read the compiled Burn manifest and embedded MSI. Never execute Setup. +param([Parameter(Mandatory)] [string] $Setup, [Parameter(Mandatory)] [string] $Msi, + [Parameter(Mandatory)] [string] $Work, [ValidateSet('x64', 'x86', 'arm64')] [string] $Arch) +$ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'installer-languages.ps1') +if (Test-Path -LiteralPath $Work) { throw "Inspection directory must be new: $Work" } +& wix burn extract $Setup -o (Join-Path $Work 'payload') -oba (Join-Path $Work 'ba') +if ($LASTEXITCODE -ne 0) { throw 'Cannot inspect compiled Setup' } +[xml] $manifest = Get-Content (Join-Path $Work 'ba/manifest.xml') -Raw -Encoding utf8 +$root = $manifest.DocumentElement +$packages = @($root.SelectNodes('//*[local-name()="MsiPackage"]')) +if ($packages.Count -ne 1) { throw 'Setup must contain exactly one MSI' } +$properties = @($packages[0].SelectNodes('*[local-name()="MsiProperty" and @Id="TRANSFORMS"]')) +if ($properties.Count -ne 1 -or $properties[0].Id -ne 'TRANSFORMS' -or $properties[0].Value -ne '[LanguageTransform]') { throw 'Missing native MSI language transform property' } +$searches = @($root.SelectNodes('//*[local-name()="SetVariable"]')) +if ($searches.Count -ne 6) { throw 'Expected six compiled language selectors' } +$map = @{} +foreach ($search in $searches) { + if ($search.Variable -ne 'LanguageTransform' -or $search.Type -ne 'string') { throw 'Unexpected language variable' } + $language = $search.Value.TrimStart(':').Replace('.mst', '') + $condition = $search.Condition + if ($condition -cne (Get-InstallerLanguageCondition $language)) { throw "Unexpected compiled selector for $language" } + foreach ($match in [regex]::Matches($condition, 'UserUILanguageID = (\d+)')) { + $id = [int] $match.Groups[1].Value + if ($map.ContainsKey($id)) { throw "Ambiguous language $id" } + $map[$id] = $language + } +} +# Exhaustive verification includes every regional sublanguage, English and all +# unsupported primary languages. No OS preference is changed to run this check. +for ($id = 0; $id -le 65535; $id++) { + $selected = if ($map.ContainsKey($id)) { $map[$id] } else { 'en' } + if ($selected -cne (Get-InstallerLanguage $id)) { throw "Incorrect fallback for LANGID $id" } +} +foreach ($culture in @('de-AT', 'fr-CA', 'es-MX', 'nl-BE', 'it-CH', 'hu-HU', 'en-AU', 'ja-JP')) { + $native = [Globalization.CultureInfo]::GetCultureInfo($culture) + $expected = if ($InstallerLanguages.ContainsKey($native.TwoLetterISOLanguageName)) { $native.TwoLetterISOLanguageName } else { 'en' } + if ((Get-InstallerLanguage $native.LCID) -ne $expected) { throw "Regional locale failure: $culture" } +} +$msiFiles = @(Get-ChildItem (Join-Path $Work 'payload') -Filter *.msi -Recurse) +if ($msiFiles.Count -ne 1 -or (Get-FileHash $msiFiles[0].FullName).Hash -ne (Get-FileHash $Msi).Hash) { throw 'Embedded MSI is not the validated offline payload' } +if ($root.SelectNodes('//*[@DownloadUrl]').Count -ne 0) { throw 'Setup must not download installation payloads' } +$bytes = [IO.File]::ReadAllBytes((Resolve-Path -LiteralPath $Setup).Path) +$pe = [BitConverter]::ToInt32($bytes, 0x3c) +$machine = [BitConverter]::ToUInt16($bytes, $pe + 4) +$expectedMachine = @{ x64 = 0x8664; x86 = 0x14c; arm64 = 0xaa64 }[$Arch] +if ($machine -ne $expectedMachine) { throw "Setup machine $machine does not match $Arch" } +Write-Output 'ok: offline Setup payload, native architecture, 65,536 LANGIDs and regional/fallback selection' From 1802d2f39cd27257aee853424525d12a125a4b00 Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:46:20 +0200 Subject: [PATCH 08/19] M10: keep installer language enumeration compatible with macOS Bash 3.2 --- packaging/macos/list-languages.applescript | 4 ++++ packaging/macos/prepare-resources.sh | 9 ++------- 2 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 packaging/macos/list-languages.applescript diff --git a/packaging/macos/list-languages.applescript b/packaging/macos/list-languages.applescript new file mode 100644 index 0000000000..b5d72d9a23 --- /dev/null +++ b/packaging/macos/list-languages.applescript @@ -0,0 +1,4 @@ +use framework "Foundation" +set languageCodes to current application's NSLocale's ISOLanguageCodes() as list +set AppleScript's text item delimiters to linefeed +return languageCodes as text diff --git a/packaging/macos/prepare-resources.sh b/packaging/macos/prepare-resources.sh index 03e7a884fd..6c71401070 100644 --- a/packaging/macos/prepare-resources.sh +++ b/packaging/macos/prepare-resources.sh @@ -13,13 +13,8 @@ cp -R "$here/resources/"* "$destination/" # of macOS languages. Only directory names are generated; every fallback is the # same contributor-original English catalog. Script-specific Chinese aliases # ensure regional preferences resolve through the matching native script. -fallbacks="$(osascript <<'APPLESCRIPT' -use framework "Foundation" -set languageCodes to current application's NSLocale's ISOLanguageCodes() as list -set AppleScript's text item delimiters to linefeed -return languageCodes as text -APPLESCRIPT -)" +# Keep AppleScript outside shell substitutions for macOS /bin/bash 3.2. +fallbacks="$(osascript "$here/list-languages.applescript")" while IFS= read -r language; do case "$language" in en|de|fr|es|nl|it|hu) continue ;; esac if [[ ! "$language" =~ ^[a-z]{2,3}$ ]]; then echo "Unexpected ISO language code: $language" >&2; exit 1; fi From 98010e75fbc29f3378928d5c8b04ab94f2224ce0 Mon Sep 17 00:00:00 2001 From: lukashold <37380729+lukashold@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:51:32 +0200 Subject: [PATCH 09/19] M10: preserve existing SCM account during Hot Folder MSI upgrades --- packaging/service-tests/windows-msi.ps1 | 2 +- packaging/windows/pdfcraft.wxs | 8 ++++++++ packaging/windows/test-msi.ps1 | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packaging/service-tests/windows-msi.ps1 b/packaging/service-tests/windows-msi.ps1 index 5db8ca3163..185d41e365 100644 --- a/packaging/service-tests/windows-msi.ps1 +++ b/packaging/service-tests/windows-msi.ps1 @@ -87,7 +87,7 @@ try { $installedMsi = $upgrade $service = Get-CimInstance Win32_Service -Filter "Name='PdfCraftHotFolder'" if ($service.State -ne 'Running' -or $service.StartMode -ne 'Auto' -or $service.StartName -ne 'NT AUTHORITY\NetworkService') { - throw 'Upgrade did not preserve the optional automatic service and its account' + throw "Upgrade did not preserve the optional automatic service and its account: state=$($service.State), startup=$($service.StartMode), account=$($service.StartName)" } if (-not (Test-Path -LiteralPath "$data/keep-user-tasks.txt")) { throw 'Upgrade deleted task data' } if ((Get-FileHash -LiteralPath "$env:ProgramFiles/PdfCraft/pdfcraft-hot-folder.exe" -Algorithm SHA256).Hash -ne $upgradeHash) { diff --git a/packaging/windows/pdfcraft.wxs b/packaging/windows/pdfcraft.wxs index 77f0b20b4e..a308793d06 100644 --- a/packaging/windows/pdfcraft.wxs +++ b/packaging/windows/pdfcraft.wxs @@ -52,6 +52,13 @@ + + + +