ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

permissions:
  contents: read

# A force-push (our normal rebase flow) otherwise leaves the superseded run to
# finish, burning ~26 runner-min and writing cache entries that compete for the
# 10GB quota. Only cancel PR runs; main-push runs must not cancel each other.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  check:
    runs-on: ubuntu-latest
    timeout-minutes: 25
    env:
      # Debug info is the largest part of a debug target dir, and this job stacks
      # a 12-16GB target on a runner with ~20GB free — see the cache note below.
      # `line-tables-only` keeps file:line in panic backtraces (how we diagnose a
      # CI-only failure) while dropping the rest. Env form, so local builds are
      # untouched. `cargo test` uses the test profile, which inherits dev.
      CARGO_PROFILE_DEV_DEBUG: line-tables-only
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
        with:
          components: clippy, rustfmt

      # Was `actions/cache` on `target/` with a key shared with `conformance` and
      # a bare `-cargo-` restore-key. Two bugs came out of that:
      #
      #  1. actions/cache does not save on an exact key hit, so whichever job
      #     finished first defined the contents. conformance won, so the cache
      #     held a RELEASE target — which this job cannot use. Measured on one
      #     commit: conformance compiled 10 crates, `check` compiled 616, from
      #     proc-macro2 up, every run. It then stacked a 12-16GB debug target on
      #     the restored release one, on ~20GB of runner disk, and died with
      #     "No space left on device" / "ld terminated with signal 7".
      #  2. The bare restore-key resurrected the previous lock's target and saved
      #     the union. actions/cache never prunes and cargo never GCs target/, so
      #     it grew monotonically (491MB -> 819MB in 16h).
      #
      # rust-cache keys per job, prunes stale artifacts before saving, and skips
      # workspace crates. `save-if` keeps PR refs from each minting their own
      # copy: caches are branch-scoped, so saving on every ref is what blew the
      # quota. PRs still restore main's cache, which is the whole win.
      #
      # `shared-key` because release.yml's `check` runs the same dev/test build
      # and wants this exact cache. Both jobs are literally named `check`, so
      # rust-cache's job-id default would have shared it by accident; naming it
      # makes that a decision, and one that survives renaming either job.
      - name: Cache cargo registry & build
        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          shared-key: dev-check
          save-if: ${{ github.ref == 'refs/heads/main' }}

      # This is the exact first command documented for a fresh contributor.
      # Keeping CI on the public entry point prevents the Makefile target and
      # the underlying Rust checks from drifting apart.
      - name: Fast contributor check
        run: make check-fast

      - name: No unguarded exit in browser-spawning code
        run: bash scripts/check-no-process-exit.sh

      - name: Internal dependency versions match workspace version
        run: bash scripts/check-internal-dep-versions.sh

      # Catches the config GAP at feature-PR time, independent of version state:
      # a new internal dep pin that isn't tracked in release-please-config.json
      # fails here long before it can go stale on a release bump. Since
      # release-please now authors its bump PR with an elevated token (see
      # release-please.yml), that PR ALSO triggers this CI on its own head SHA,
      # so a stale pin is caught pre-merge instead of post-tag.
      - name: Audit release-please config completeness
        run: python3 scripts/release/audit_release_please_config.py

      # The guards above are load-bearing; this regression suite fails red if a
      # future change weakens them (drift/missing-entry/anti-vacuity/invalid
      # jsonpath), instead of letting them pass vacuously.
      - name: Guard regression tests
        run: python3 scripts/release/test_guards.py

      # No standalone `cargo build --workspace --all-targets`: the clippy step
      # inside `make check-fast` runs with `--all-targets -D warnings`
      # (compile-checks every target, incl. the
      # criterion bench + example) and `cargo test` builds & runs all test
      # targets. A separate dev-profile build only recompiles the workspace for
      # artifacts CI never runs. (Kept in release.yml as a pre-publish gate.)
      - name: Test
        run: cargo test --workspace

      # `cargo test --workspace` unifies features, so crw-server is always built
      # WITH `cdp` (crw-cli enables it). Run the crate on its own default feature
      # set too, so the CDP-less build is really exercised — that is the branch
      # where `/v1/capabilities` must report no renderers and no screenshot.
      - name: Test crw-server without the cdp feature
        run: cargo test -p crw-server

  sdk-ts:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    strategy:
      fail-fast: false
      matrix:
        node-version: [22, 24]
    defaults:
      run:
        working-directory: sdks/typescript
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: ${{ matrix.node-version }}

      - name: Install (reproducible)
        run: npm ci

      - name: Build
        run: npm run build

      - name: Test
        run: npm test

      # The launcher's SHA256SUMS parser is hand-rolled because Windows ships no
      # sha256sum, and Windows is the one platform with no prebuilt package, so
      # every Windows user depends on it. No deps, node:test is stdlib.
      - name: Test crw-mcp launcher
        working-directory: mcp/crw-mcp
        run: npm test

  # Firecrawl v2 conformance gate (conformance/, issue #62). Drives the
  # deterministic corpus against a live `crw serve` and diffs each response's
  # SHAPE against the committed golden Firecrawl fixtures. `compare.py` only
  # flags golden keys MISSING from crw's output (additive fields are invisible),
  # so this catches accidental removal/rename of a contract field — the exact
  # regression our additive Phase-0/1 work must never introduce.
  #
  # Tier-1 cases hard-fail CI; Tier-2 (LLM-dependent extract/json/summary, PDF
  # parsers) are reported but non-gating, so no LLM key is needed. `search_basic`
  # is excluded (CONFORMANCE_SKIP): it needs a live SearXNG fanning out to
  # third-party engines, which rate-limit CI IPs to zero results and flake the
  # gate on an external dependency, not on crw. The scrape/map/crawl/batch/parse
  # cases hit stable targets (example.com, firecrawl.dev, a w3.org PDF) and are
  # deterministic. (Run `compare` locally with a SearXNG up to gate search too.)
  conformance:
    runs-on: ubuntu-latest
    # Generous: the first run after a cache-key change rebuilds the whole tree
    # with `lto = true, codegen-units = 1`, i.e. a single-threaded fat-LTO link.
    # The uncached release binaries measure ~10 min, so this is ~2.5x that.
    timeout-minutes: 40
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable

      # See the note on `check`: these two jobs shared one key while building
      # different profiles. rust-cache keys on the job id, so they no longer
      # collide, and this job stops handing `check` a release target it can't use.
      - name: Cache cargo registry & build
        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          # v9 flipped this default to false; keep the v8 behaviour so the
          # uv cache stays trimmed and does not crowd the Rust caches out.
          prune-cache: true

      - name: Build crw
        run: cargo build --release --bin crw

      - name: Start crw serve
        run: |
          ./target/release/crw serve --host 127.0.0.1 --port 3000 &
          echo $! > /tmp/crw.pid
          for i in $(seq 1 30); do
            if curl -sf http://localhost:3000/health >/dev/null; then echo "crw up"; break; fi
            # Without this the loop just falls through after 30 failed probes and
            # `compare` runs against a dead server, reporting a confusing diff
            # instead of "the server never came up".
            if [ "$i" -eq 30 ]; then echo "::error::crw serve never became healthy"; exit 1; fi
            sleep 1
          done

      - name: Conformance — golden-fixture shape diff
        working-directory: conformance
        env:
          CRW_URL: http://localhost:3000
          CONFORMANCE_SKIP: search_basic
        run: uv run ./run.sh compare

      - name: Stop crw serve
        if: always()
        run: kill "$(cat /tmp/crw.pid)" 2>/dev/null || true