feat(processors): index .doc/.docx, .xls/.xlsx and .msg documents

Dockerfile

@@ -6,10 +6,18 @@ COPY --from=ghcr.io/astral-sh/uv:0.11.31@sha256:ecd4de2f060c64bea0ff8ecb182ddf46
 # 1. curl (required for container healthcheck probes)
 # 2. git (required for caldav dependency from git)
 # 3. sqlite for development with token db
+# 4. libreoffice-writer/-calc: the only readers for legacy .doc/.xls, and the
+#    renderer behind the .doc/.docx PDF rendition that gives those formats page
+#    numbers and highlight geometry. Writer and Calc specifically, not the
+#    metapackage -- Impress/Draw/Base would roughly double the layer for
+#    formats we do not index. The code degrades to "no processor for type" when
+#    the binary is absent, so a slimmer image variant stays possible.
 RUN apt update && apt install --no-install-recommends --no-install-suggests -y \
     curl \
     git \
     tesseract-ocr \
+    libreoffice-writer \
+    libreoffice-calc \
     sqlite3 && apt clean
 
 # Build in /src, run in /app, keep the venv in /opt/venv. The three have

docs/configuration.md

@@ -723,6 +723,7 @@ shorter OCR ceiling:
 
 ```dotenv
 DOCUMENT_PARSE_TIMEOUT_SECONDS=120    # Wall-clock cap per isolated parse (default: 120)
+DOCUMENT_OFFICE_TIMEOUT_SECONDS=120   # Wall-clock cap per LibreOffice conversion: .doc/.docx -> pdf, .xls -> xlsx (default: 120)
 DOCUMENT_OCR_TIMEOUT_SECONDS=180      # OCR backend request timeout (default: 180)
 DOCUMENT_MAX_PDF_SIZE_MB=50           # Pre-parse size cap; 0 disables (default: 50)
 DOCUMENT_PARSE_PAGE_WINDOW=100        # Pages per extraction window; 0 disables (default: 100)
@@ -751,6 +752,18 @@ A PDF larger than `DOCUMENT_MAX_PDF_SIZE_MB` fails fast with reason `oversize`
 of being handed to the tiers, where a 40+ MB scan would otherwise burn the full
 OCR timeout for zero recovered text.
 
+The same cap is applied twice to a `.doc`/`.docx`: once to the source before it
+is downloaded, and again to the PDF LibreOffice renders from it. Rendering is
+not size-preserving — a modest document of dense vector figures can render much
+larger — so a source under the cap can still produce a rendition over it, and
+that rendition is what the parse tiers would have to hold.
+
+A LibreOffice conversion also pays `DOCUMENT_OFFICE_TIMEOUT_SECONDS` before the
+parse cap applies, and it takes a slot from `DOCUMENT_PARSE_PROCESS_SLOTS` while
+it runs: `soffice` holds the source and the rendered output at once, so it is
+bounded by the same limiter as the parse workers rather than being allowed to
+start one process per concurrent ingest task.
+
 **Sizing the cap for a tenant.** Two metrics make the corpus visible instead of
 requiring a manual crawl:
 

nextcloud_mcp_server/config.py

@@ -248,6 +248,11 @@
     # PDF parse isolation (OOM guard)
     "document_pdf_graphics_limit": 1000,
     "document_parse_timeout_seconds": 120.0,
+    # Wall-clock cap on one LibreOffice conversion (.doc/.docx -> pdf,
+    # .xls -> xlsx). Separate from document_parse_timeout_seconds: a rendition
+    # pays this first and then the parse cap, and the two are tuned against
+    # different costs (page layout vs. table detection).
+    "document_office_timeout_seconds": 120.0,
     # Optional wall-clock cap (seconds) on the SYNCHRONOUS parse inside the
     # nc_webdav_read_file MCP tool. None (default) = disabled: an interactive read
     # is bounded only by the underlying processor timeout (DOCLING_TIMEOUT /
@@ -603,6 +608,11 @@ def _resolve_settings_files() -> list[str]:
         Validator("CHUNKING_CONFIG_VERSION", gte=1),
         Validator("DOCUMENT_PARSE_TIMEOUT_SECONDS", gte=1),
         Validator("DOCUMENT_OCR_TIMEOUT_SECONDS", gte=1),
+        # Same floor as its siblings: 0 or negative reaches anyio.fail_after in
+        # _libreoffice.convert and expires every conversion immediately, so the
+        # misconfiguration surfaces as "every .doc/.docx fails to parse" rather
+        # than as a startup error naming the setting.
+        Validator("DOCUMENT_OFFICE_TIMEOUT_SECONDS", gte=1),
         # DOCUMENT_OCR_MODE is normalised + membership-checked in
         # Settings.__post_init__ via _enum_fields (case-insensitive, like
         # DOCUMENT_OCR_PROVIDER) — no strict dynaconf Validator here, so
@@ -1235,6 +1245,11 @@ class Settings:
     # float so a fractional DOCUMENT_PARSE_TIMEOUT_SECONDS is honoured, matching
     # anyio.move_on_after's float seconds.
     document_parse_timeout_seconds: float = 120.0
+    # Wall-clock cap on one LibreOffice conversion (.doc/.docx -> pdf,
+    # .xls -> xlsx). Separate from document_parse_timeout_seconds because a
+    # rendition pays this first and the parse cap afterwards, and the two bound
+    # different costs -- page layout vs. table detection.
+    document_office_timeout_seconds: float = 120.0
     # Optional cap (seconds) on the synchronous parse in the nc_webdav_read_file
     # tool. None = disabled (bounded only by the processor timeout). When set,
     # anyio.fail_after aborts a slow interactive convert and the tool returns

nextcloud_mcp_server/document_processors/__init__.py

@@ -1,12 +1,20 @@
 """Document processing plugins for extracting text from various file formats."""
 
+import logging
+
 from nextcloud_mcp_server.config import get_settings
 
+from . import _libreoffice
 from .base import DocumentProcessor, ProcessingResult, ProcessorError
+from .msg import MsgProcessor
 from .ocr import OcrProcessor
+from .office import OfficeDocumentProcessor
 from .pymupdf import PyMuPDFProcessor
 from .pypdfium2_fast import Pypdfium2FastProcessor
 from .registry import ProcessorRegistry, get_registry
+from .spreadsheet import SpreadsheetProcessor
+
+logger = logging.getLogger(__name__)
 
 # Register processors at module initialization. The tiered PDF pipeline selects
 # by tier (not priority): Pypdfium2FastProcessor is the ``fast`` tier,
@@ -37,13 +45,33 @@
     priority=1,
 )
 
+# Office and email formats. Priority 15 puts them above the optional
+# Unstructured processor (10), which also claims these types but flattens their
+# structure, while leaving 20 for docling. Each reads its format the way that
+# measured best rather than sharing one route -- see the module docstrings.
+_office_timeout = _settings.document_office_timeout_seconds
+_registry.register(SpreadsheetProcessor(timeout=_office_timeout), priority=15)
+_registry.register(MsgProcessor(), priority=15)
+if _libreoffice.LIBREOFFICE_AVAILABLE:
+    _registry.register(OfficeDocumentProcessor(timeout=_office_timeout), priority=15)
+else:
+    # Not an error: the API image has no reason to carry LibreOffice. Leaving
+    # the processor unregistered means .doc/.docx report "no processor for
+    # type" once, rather than every document failing mid-parse.
+    logger.info(
+        "LibreOffice not found; .doc/.docx indexing is unavailable in this image"
+    )
+
 __all__ = [
     "DocumentProcessor",
     "ProcessingResult",
     "ProcessorError",
     "ProcessorRegistry",
     "get_registry",
+    "MsgProcessor",
+    "OfficeDocumentProcessor",
     "PyMuPDFProcessor",
     "Pypdfium2FastProcessor",
     "OcrProcessor",
+    "SpreadsheetProcessor",
 ]

nextcloud_mcp_server/document_processors/_libreoffice.py

@@ -0,0 +1,145 @@
+"""LibreOffice headless conversion, used to reach formats nothing else reads.
+
+Legacy binary Office documents (``.doc``, ``.xls``) have no working pure-Python
+reader: markitdown raises ``UnsupportedFormatException`` and docling-serve
+returns ``status=failure``. LibreOffice is the only thing that opens them, and
+it is already present in several of our images.
+
+Two conversion targets, chosen per source format rather than uniformly:
+
+* ``pdf`` -- for word-processor documents, whose page layout *is* their
+  structure. The rendition is a real PDF, so the existing fast/structured
+  ladder, ``page_boundaries`` and ``pdf_highlighter`` all work on it unchanged.
+* ``xlsx`` -- for legacy spreadsheets, which are then read cell-by-cell. A
+  spreadsheet must NOT go to PDF: measured on a real workbook, PDF rendering
+  paginated it into 25 mixed-orientation pages and recalled only 63.8% of the
+  tokens a direct cell read recovers, because LibreOffice honours the print
+  layout and drops whatever falls outside it.
+"""
+
+import logging
+import pathlib
+import shutil
+import tempfile
+from typing import Final
+
+import anyio
+
+from nextcloud_mcp_server.config import get_settings
+
+from ._isolation import parse_process_limiter
+
+logger = logging.getLogger(__name__)
+
+# Resolved once at import, like TesseractProcessor's availability probe, so a
+# deployment without LibreOffice degrades to "no processor for this type"
+# rather than failing per document.
+SOFFICE_BIN: Final[str | None] = shutil.which("soffice") or shutil.which("libreoffice")
+LIBREOFFICE_AVAILABLE: Final[bool] = SOFFICE_BIN is not None
+
+
+class LibreOfficeError(Exception):
+    """Raised when a LibreOffice conversion fails or produces no output."""
+
+
+async def convert(
+    content: bytes,
+    filename: str,
+    target: str,
+    timeout_seconds: float = 120.0,
+) -> bytes:
+    """Convert ``content`` to ``target`` ("pdf" / "xlsx") and return the bytes.
+
+    The cap is taken as a value rather than left to the caller's own
+    ``fail_after`` because expiry is translated here into a
+    :class:`LibreOfficeError`: callers already handle that one exception type,
+    and a bare ``TimeoutError`` escaping into the processor would bypass the
+    "conversion failed" path each of them implements.
+
+    Args:
+        content: Source document bytes.
+        filename: Source filename -- LibreOffice picks its import filter from
+            the extension, so a name without one converts as the wrong format.
+        target: LibreOffice output filter name.
+        timeout_seconds: Wall-clock cap; the process is killed past it.
+
+    Raises:
+        LibreOfficeError: LibreOffice is absent, exits non-zero, times out, or
+            writes no output file.
+    """
+    if SOFFICE_BIN is None:
+        raise LibreOfficeError("LibreOffice (soffice) is not installed")
+
+    suffix = pathlib.Path(filename).suffix
+    if not suffix:
+        raise LibreOfficeError(
+            f"cannot convert {filename!r}: no extension to select an import filter"
+        )
+
+    with tempfile.TemporaryDirectory(prefix="lo-convert-") as tmp:
+        tmpdir = pathlib.Path(tmp)
+        src = tmpdir / f"source{suffix}"
+        src.write_bytes(content)
+        outdir = tmpdir / "out"
+        outdir.mkdir()
+
+        # -env:UserInstallation gives this invocation a private profile
+        # directory. Without it every concurrent soffice shares one profile and
+        # the second one either blocks on the lock or exits 0 having written
+        # nothing -- which would surface as a random empty-output failure under
+        # parallel ingest rather than as anything diagnosable.
+        profile = tmpdir / "profile"
+        argv = [
+            SOFFICE_BIN,
+            f"-env:UserInstallation=file://{profile}",
+            "--headless",
+            "--norestore",
+            "--convert-to",
+            target,
+            "--outdir",
+            str(outdir),
+            str(src),
+        ]
+
+        # Bounded by the same limiter as the isolated PDF parse. A LibreOffice
+        # process is the heaviest thing the ingest path spawns -- it holds the
+        # source document and the rendered output at once -- so leaving it
+        # unbounded would let a folder of .doc files start one soffice per
+        # concurrent task and exhaust the pod's memory, which is exactly what
+        # that limiter exists to prevent for the (lighter) parse workers.
+        # Acquired and released here, before the delegated parse acquires it in
+        # turn, so the two never nest.
+        settings = get_settings()
+        limiter = parse_process_limiter(settings.document_parse_process_slots)
+        try:
+            async with limiter:
+                with anyio.fail_after(timeout_seconds):
+                    result = await anyio.run_process(argv, check=False)
+        except TimeoutError as exc:
+            raise LibreOfficeError(
+                f"LibreOffice timed out after {timeout_seconds}s converting {filename!r}"
+            ) from exc
+
+        if result.returncode != 0:
+            stderr = result.stderr.decode("utf-8", "replace").strip()[:500]
+            raise LibreOfficeError(
+                f"LibreOffice exited {result.returncode} for {filename!r}: {stderr}"
+            )
+
+        # soffice exits 0 on an unreadable input while writing nothing, so the
+        # output file's existence -- not the return code -- is the real check.
+        produced = sorted(outdir.iterdir())
+        if not produced:
+            raise LibreOfficeError(
+                f"LibreOffice produced no {target} output for {filename!r}"
+            )
+
+        data = produced[0].read_bytes()
+        logger.debug(
+            "LibreOffice converted %s (%d bytes) -> %s (%d bytes)",
+            filename,
+            len(content),
+            target,
+            len(data),
+        )
+        return data