python.yml

name: Python gates

# Path filtered on purpose: a documentation commit must not spend runner minutes
# on the backend gates.
on:
  push:
    paths:
      - 'backend/**'
      - 'scripts/**'
      # Two reasons for the second line, and both are load bearing.
      # backend/tests/test_corpus_terms.py and backend/tests/test_ocr_quality.py
      # load scripts/dev/build_corpus.py at run time and read FILES,
      # _searchable_text and UNIQUE_TERMS out of it, so a commit that touches
      # only that script can turn the suite red. integration.yml has carried the
      # file in its path list since phase 3 for the same reason. Without this
      # line the breakage surfaces on the next commit that happens to touch
      # backend/** and is blamed on the wrong one. And since this workflow also
      # lints scripts/ with the rule set of the backend, a filter narrower than
      # the thing it checks would be a gate that cannot see its own subject.
      - '.github/workflows/python.yml'
  pull_request:
    paths:
      - 'backend/**'
      - 'scripts/**'
      - '.github/workflows/python.yml'
  # The measurement job below runs on these two only. It costs minutes and it
  # guards no regression, it produces a number, so it has no business in the loop
  # that has to answer a pull request quickly.
  workflow_dispatch:
  schedule:
    - cron: '17 4 * * 1'

permissions:
  contents: read

# One run per branch. A push on top of a push makes the older run's verdict
# worthless, and on a repository with a runner budget worthless minutes are the
# ones that make the next pull request wait.
concurrency:
  group: python-${{ github.ref }}
  cancel-in-progress: true

defaults:
  run:
    working-directory: backend

jobs:
  gates:
    name: ruff, format, pyright, vulture, pytest
    runs-on: ubuntu-24.04
    # Measured over the sixteen most recent green runs, read on 03.09.2026: 12
    # seconds with a warm uv cache, up to two minutes and twelve seconds when the
    # cache misses and the locked environment is downloaded. Fifteen minutes is
    # roughly seven times the worst of those, and it is a deadline where there
    # was none: the default is 360 minutes.
    timeout-minutes: 15
    steps:
      # Actions are pinned to commit SHAs, never to tags: a tag can be moved and
      # would hand a third party arbitrary code execution in this build.
      - name: Check out the repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          # Same uv release that produced backend/uv.lock, so --frozen means the
          # same resolution locally and in CI.
          version: '0.11.7'
          enable-cache: true
          cache-dependency-glob: backend/uv.lock

      - name: Install the locked environment
        run: uv sync --frozen

      - name: Lint
        run: uv run ruff check .

      - name: Format check
        run: uv run ruff format --check .

      - name: Type check
        run: uv run pyright

      - name: Dead code
        run: uv run vulture src tests --min-confidence 80

      # The gates above all run with working-directory: backend, and
      # [tool.pyright] include and the vulture arguments name src and tests, so
      # until this step existed the Python files under scripts/ fell through
      # every one of the four. That is not a small corner: scripts/dev carries
      # build_corpus.py, which the suite loads at run time, and
      # compound_probe.py, whose output is the measurement ground of the German
      # analysis chain. A typo in either of them reached the documentation
      # before anything looked at it.
      #
      # --config is not decoration. ruff resolves its configuration by walking
      # up from the file it is checking, there is no pyproject.toml at the
      # repository root, and without the option ruff would silently fall back to
      # its own small default rule set: the step would be green and would mean
      # almost nothing. Naming the file the backend already uses is what makes
      # this the same rule set, not a second one that can drift.
      #
      # Deliberately ruff only, no pyright and no vulture, and that is a
      # documented limit rather than an oversight. A dev script reaches its
      # dependencies over sys.path rather than over an installed package, so
      # pyright reports unresolved imports for scripts/dev/vector_distances.py
      # that say nothing about the code, and vulture would flag every entry
      # point a human calls by hand. Both are worth revisiting if scripts/ ever
      # grows a package; ruff and the formatter are what these files need today.
      - name: Lint the scripts, with the rule set of the backend
        run: uv run ruff check --config pyproject.toml ../scripts

      - name: Format check the scripts
        run: uv run ruff format --config pyproject.toml --check ../scripts

      # The gate of DI-06.1-31, named after what it prevents and run before the
      # suite it is also part of. The duplication is the point: the Office bug of
      # plan 06.1-19 was a "File damaged" verdict on every DOCX, XLSX and PPTX of
      # a machine with enough cores, it survived five phases because no job here
      # could reach it, and a case that reappears in a line reading "212 passed"
      # is a case that can be dropped again without anybody noticing. This step
      # carries its own falsification: the case fails on purpose in a child that
      # runs the state before commit debb395, so a green step means the trap is
      # reachable AND the fix holds, not merely that nothing was tried.
      - name: The many core address space trap of the Office path (DI-06.1-31)
        run: uv run pytest -q tests/test_sandbox.py -k many_core_trap --no-header

      - name: Tests
        run: uv run pytest -q

  # The only job in this repository that runs on arm64. The target hardware of
  # this app is a 4 GB ARM board, and the decision it feeds is whether the
  # extraction child lives across files or is started per file: at 100.000 files
  # a start cost of half a second is 14 hours, and one of two seconds is 55. That
  # was an assumption in the phase research (A11), and an assumption about the
  # slowest machine in the fleet, measured on the fastest, is not evidence.
  extract-bench-arm:
    name: extraction start cost on arm64
    runs-on: ubuntu-24.04-arm
    if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
    # This job had never run: it fires on dispatch and on the Monday schedule
    # only, and every recorded run of this workflow skipped it, so there was
    # nothing to derive a deadline from. It was therefore dispatched once on
    # 03.09.2026 rather than estimated, and it took one minute and twelve
    # seconds for the thousand samples (run 33766223084). Twenty minutes is
    # sixteen times that, which leaves room for a slower runner and for a larger
    # sample count, and is still a deadline instead of the default 360.
    timeout-minutes: 20
    steps:
      - name: Check out the repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: '0.11.7'
          enable-cache: true
          cache-dependency-glob: backend/uv.lock

      - name: Install the locked environment
        run: uv sync --frozen

      # 1000 cycles because the figure that matters is p95, and a p95 over a
      # handful of samples is a rumour. The job prints numbers only, never a path
      # and never a document.
      - name: Measure the process start cost
        run: uv run python -m findling.extract.bench --spawns 1000