feat: complete and harden the off-chain LLM resolver lifecycle

.dockerignore

@@ -1,6 +1,8 @@
 node_modules
 test-program
 keys
-.env
+.env*
 .git
 *.md
+**/*.test.ts
+src/e2e

.env.example

@@ -3,12 +3,32 @@ GEMINI_API_KEY=
 
 # Gemini model for resolution calls
 GEMINI_MODEL=gemini-flash-latest
+GEMINI_REQUEST_TIMEOUT_MS=30000
 
 # Google-Search grounding (web search). Needs a paid tier — grounded calls
 # 429 (RESOURCE_EXHAUSTED) on the free tier. "false" runs on the free tier
 # using the model's training knowledge only (no live web verification).
 GEMINI_GROUNDING=false
 
+# Production deep-research adapter. Unlike Gemini, Parallel is asynchronous:
+# the resolver persists round -> run IDs so restarts do not duplicate paid
+# research and completed verdicts survive on-chain submission retries.
+PARALLEL_API_KEY=
+PARALLEL_PROCESSOR=pro
+PARALLEL_STATE_PATH=.state/parallel-tasks.json
+PARALLEL_REQUEST_TIMEOUT_MS=10000
+PARALLEL_RETRY_DELAY_MS=60000
+PARALLEL_MAX_ATTEMPTS=3
+PARALLEL_STATE_RETENTION_MS=604800000
+
+# Resolution Specs are referenced on-chain as:
+# ar://<43-character-Arweave-transaction-id>#sha256=<64-character-hex-digest>
+# The resolver fetches from this gateway, enforces the byte limit, and verifies
+# the exact downloaded bytes before calling the model.
+ARWEAVE_GATEWAY=https://arweave.net
+SPEC_FETCH_TIMEOUT_MS=10000
+SPEC_MAX_BYTES=65536
+
 # Solana RPC endpoint (devnet for the demo, localnet for e2e tests)
 RPC_URL=https://api.devnet.solana.com
 
@@ -21,5 +41,31 @@ PROGRAM_ID=
 # Poll interval in milliseconds
 POLL_INTERVAL_MS=10000
 
-# LLM adapter: "gemini" or "stub" (stub returns a fixed verdict; for tests)
+# Permissionless recovery/finalization. Enable only against the current Opal
+# layout: timed-out PendingLLM rounds recover to challengeable Unresolvable,
+# then expired undisputed and LLM assertions are finalized.
+ENABLE_FINALIZE_CRANK=false
+
+# LLM adapter: "gemini" or "parallel". Test stubs cannot be selected by the
+# production worker; tests inject them directly without signing live verdicts.
 LLM_ADAPTER=gemini
+
+# Production supervision. The worker atomically refreshes this file after each
+# cycle; Docker HEALTHCHECK fails when it is degraded or stale. It exits after
+# repeated top-level cycle failures so the host restart policy can recover it.
+HEALTH_FILE_PATH=/tmp/opal-resolver-health.json
+HEALTH_MAX_AGE_MS=60000
+# Force the worker process to exit if an RPC/provider operation makes no
+# observable scan/item/crank progress. Docker can then apply its restart policy.
+WORKER_STALL_TIMEOUT_MS=120000
+MAX_CONSECUTIVE_CYCLE_FAILURES=6
+MIN_RESOLVER_BALANCE_LAMPORTS=20000000
+# Production Parallel defaults to requiring at least one hour on-chain. The
+# worker fails preflight if ProtocolConfig would recover the round sooner.
+# Gemini derives its default minimum from poll + spec-fetch + model timeouts,
+# plus a 30-second transaction/submission safety margin.
+# Override only when the selected processor has a deliberately smaller budget.
+# MIN_CHAIN_LLM_TIMEOUT_SECONDS=3600
+
+# Optional Slack-compatible JSON webhook for cycle failure/recovery alerts.
+ALERT_WEBHOOK_URL=

.github/workflows/ci.yml

@@ -0,0 +1,75 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  resolver:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+      - run: bun install --frozen-lockfile
+      # SPL Token is dev-only and currently has no patched bigint-buffer release;
+      # keep that single reviewed advisory isolated while failing on any new one.
+      - run: bun audit --audit-level=low --ignore=GHSA-3gc7-fjrx-p6mg
+      - run: bun run test
+      - run: bunx tsc --noEmit
+      - name: Bundle production entry points
+        run: >-
+          bun build src/main.ts src/healthcheck.ts
+          --target bun --outdir /tmp/opal-resolver-build
+
+  dummy-program:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: test-program
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install pinned Rust toolchain
+        run: rustup toolchain install 1.89.0 --profile minimal --component rustfmt,clippy
+      - run: cargo fmt --all -- --check
+      - run: cargo test --workspace --all-features --all-targets
+      - run: cargo clippy --workspace --all-features --all-targets -- -D warnings
+
+  dummy-e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+      - name: Install pinned Rust toolchain
+        run: rustup toolchain install 1.89.0 --profile minimal
+      # AVM 1.1.2's installer dependencies require Rust 1.91. The program still
+      # builds with 1.89 via test-program/rust-toolchain.toml.
+      - name: Install AVM bootstrap toolchain
+        run: rustup toolchain install 1.91.0 --profile minimal
+      - name: Install Anchor version manager
+        run: >-
+          cargo +1.91.0 install avm
+          --git https://github.com/solana-foundation/anchor
+          --rev 24035e2b0035c87e321acc1c05f97793829a87f1
+          --locked
+      - name: Install pinned Anchor CLI
+        run: avm install 1.1.2
+      - run: avm use 1.1.2
+      - run: bun install --frozen-lockfile
+      # Anchor installs the Solana version from test-program/Anchor.toml on the
+      # first build. Register its destination before that step so the installer
+      # can immediately activate agave-install, solana, and the test validator.
+      - name: Register Solana toolchain path
+        run: echo "$HOME/.local/share/solana/install/active_release/bin" >> "$GITHUB_PATH"
+      # Anchor.toml pins Solana 3.1.13; Anchor installs/dispatches that toolchain
+      # before building the fixed-ID dummy program.
+      - run: bun run e2e:build
+      - run: bun run e2e

.gitignore

@@ -17,6 +17,8 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 
 # dotenv environment variable files
 .env
+.env.production
+.state/
 .env.development.local
 .env.test.local
 .env.production.local

Dockerfile

@@ -1,13 +1,17 @@
-FROM oven/bun:1 AS runtime
+FROM oven/bun:1@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 AS runtime
 WORKDIR /app
 
 COPY package.json bun.lock ./
 RUN bun install --frozen-lockfile --production
 
 COPY src ./src
 
+RUN mkdir -p /var/lib/opal-resolver && chown bun:bun /var/lib/opal-resolver
+
 # No HTTP surface — this is a long-running worker. Configuration and secrets
 # come from the environment (docker run --env-file .env); the resolver keypair
 # is mounted, never baked into the image.
 USER bun
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+  CMD ["bun", "run", "src/healthcheck.ts"]
 CMD ["bun", "run", "src/main.ts"]

README.md

@@ -5,11 +5,14 @@ The trusted LLM resolver service for [Opal](https://github.com/jewl-labs/opal) 
 A single long-running Bun worker that:
 
 1. **Scans** the Opal program for unresolved `LlmResolutionRound` PDAs (`getProgramAccounts`, filtered on the account discriminator + outcome byte `255` at offset 72; account size 98 bytes). Chain state is the work queue — there is no database.
-2. **Reads** the disputed assertion's statement and calls **Gemini** to judge it against its Resolution Spec. Google-Search grounding (live web) is optional via `GEMINI_GROUNDING` — off by default because it requires a paid Gemini tier.
+2. **Loads and verifies** the assertion's Arweave Resolution Spec from its on-chain reference, then applies it with either **Gemini** (fast/demo) or **Parallel deep research** (live-web production path). Gemini Google-Search grounding remains optional and off by default because it requires a paid Gemini tier.
 3. **Submits** the verdict on-chain via `submit_llm_resolution`, signed by the resolver keypair (`ProtocolConfig.resolver`). Verdicts are `True (0)`, `False (1)`, or `Unresolvable (3)` — the program rejects `TooEarly (2)` per ADR-0005.
+4. Optionally **cranks recovery and finalization** when `ENABLE_FINALIZE_CRANK=true`: timed-out `PendingLLM` rounds recover to challengeable `Unresolvable`, while expired undisputed assertions and expired True/False/Unresolvable verdicts finalize permissionlessly. Enable this only against the current Opal account layout.
 
 Idempotency is on-chain: a posted verdict drops out of the next scan, and the program's `PendingLLM` guard rejects any second submission. Chain state is both the queue and the dedup.
 
+Parallel runs asynchronously. A small local registry persists the paid run ID and completed verdict for each round; it is not a second work queue or source of truth. This prevents duplicate research after restarts and reuses the same verdict if on-chain submission must retry.
+
 ## Run
 
 ```bash
@@ -20,33 +23,87 @@ bun start
 ```
 
 `GEMINI_GROUNDING=false` (the default) resolves from the model's training knowledge with no live web access; set it `true` only on a paid Gemini tier (grounded calls 429 on the free tier).
+Gemini calls fail after `GEMINI_REQUEST_TIMEOUT_MS` (30 seconds by default) so a stalled provider cannot block the polling loop indefinitely.
+
+For live-web deep research without Gemini grounding, select Parallel:
+
+```dotenv
+LLM_ADAPTER=parallel
+PARALLEL_API_KEY=<secret>
+PARALLEL_PROCESSOR=pro
+PARALLEL_STATE_PATH=.state/parallel-tasks.json
+```
+
+The first poll creates one research run and later polls retrieve its status. The completed structured verdict is cached until the on-chain round leaves the queue. Keep the state path on durable storage in production.
+
+The deterministic stub adapter is available only to tests and cannot be selected by the production worker.
+
+## Resolution Specs
+
+Every resolvable assertion must put this value in its on-chain `auxiliary_hash` field:
+
+```text
+ar://<43-character-Arweave-transaction-id>#sha256=<64-character-lowercase-hex>
+```
+
+The combined reference is at most 128 bytes, matching the account field. The resolver fetches only that transaction from `ARWEAVE_GATEWAY`, caps the response size, verifies SHA-256 over the exact downloaded bytes, requires non-empty UTF-8, and only then sends the spec to the model. A missing, malformed, unavailable, oversized, or hash-mismatched spec is retried later; no verdict is invented or posted.
+
+After uploading a spec, generate the exact on-chain reference from the returned transaction ID and the same local file:
+
+```bash
+bun run spec:reference -- <arweave-transaction-id> ./resolution-spec.md
+```
 
 Flags:
 
-- `--manual <assertion_pubkey> <outcome_code>` — post one verdict by hand and exit (demo fallback).
+- `--manual <assertion_id> <outcome_code>` — post one verdict by hand and exit (demo fallback).
 - `--once` — single scan pass instead of the 10s loop.
 
+Governance can rotate a compromised or retired resolver key without redeploying:
+
+```bash
+bun run resolver:rotate -- <deployment-config.json> <new-resolver-pubkey>
+```
+
+After rotation, fund the new key with SOL, update `RESOLVER_KEYPAIR_PATH`, run one `--once` health pass, and then retire the old key.
+
 ## Test
 
 ```bash
-bun run test    # unit: verdict parsing, injection screening, discriminators (no external deps)
+bun run test    # unit, failure-boundary, script, and deterministic stress tests
+bun run e2e:build # build the fixed-ID dummy program without syncing local keys
 bun run e2e     # end-to-end against the dummy anchor program on a local validator
+bun run e2e:opal # cross-repo compatibility test against the real Opal program
 ```
 
 `bun run test` needs nothing external. The e2e runs against `test-program/` — a minimal Anchor **1.1.2** program with the same 98-byte round layout as Opal, so the full scan → resolve → submit loop is proven without touching the real program. It requires the Solana CLI (`solana-test-validator`) and Anchor 1.1.2, and the dummy program built first:
 
 ```bash
-cd test-program && anchor build && cd ..
+bun run e2e:build
 bun run e2e
 ```
 
+The real-program compatibility suite consumes build artifacts from an Opal checkout, so it does not hard-code a developer's filesystem layout or duplicate the on-chain program:
+
+```bash
+cd /path/to/opal && anchor build
+cd /path/to/opal-resolver
+OPAL_E2E_SO_PATH=/path/to/opal/target/deploy/opal.so \
+OPAL_E2E_IDL_PATH=/path/to/opal/target/idl/opal.json \
+bun run e2e:opal
+```
+
+`bun run e2e:opal` fails immediately with a configuration error if either artifact path is omitted.
+
+It launches a throwaway upgradeable local deployment and initializes the real `ProtocolConfig`. It verifies payout conservation for direct True, False, and Unresolvable verdicts; resolver timeout → permissionless Unresolvable recovery → no-fault finalization; undisputed finalization; and immediate old-key rejection after resolver rotation. Cross-repo CI should run this command whenever either side changes its account layout or instruction contract.
+
 > Note: plain `bun test` (no script) runs **both** suites, so it carries the same e2e prerequisites. Use `bun run test` for the dependency-free unit suite.
 
 ## Running it
 
 For a demo or a one-off, just run it locally — `bun start` (see [Run](#run)); no container needed.
 
-For **persistent operation** it needs to stay up whenever a dispute exists (an unresolved assertion has no on-chain timeout), so long-term it runs as a single always-on worker. The Dockerfile packages exactly that — no HTTP surface, two secrets (resolver keypair + Gemini key) injected via env:
+For **persistent operation** it runs as a single always-on worker. The Dockerfile packages exactly that—no HTTP surface, with the resolver keypair and selected provider API key injected at runtime:
 
 ```bash
 docker build -t opal-resolver .
@@ -55,8 +112,16 @@ docker run --env-file .env opal-resolver
 
 Any container host works (Fly.io, Railway, a small VPS); restart-on-crash comes from the host's restart policy.
 
+For the supervised, fail-closed production setup (preflight, file healthcheck,
+structured cycle metrics, alerts, graceful shutdown, key rotation, and incident
+response, plus a stalled-progress watchdog that exits for host restart), see
+[docs/production-runbook.md](docs/production-runbook.md).
+
+For rehearsal, stream deployment, normal operation, and fallback commands, see
+[docs/demo-runbook.md](docs/demo-runbook.md).
+
 ## Security notes
 
 - The resolver key is a hot key. On-chain it can only post challengeable verdicts — it is deliberately **not** the protocol authority.
-- Statements are untrusted input: they are wrapped in tags, screened for prompt injection, and the model is instructed to return `Unresolvable` on any attempt to steer it.
+- Statements and specs are untrusted input: they are isolated in separate tags, the statement is screened for prompt injection, and the model is instructed to return `Unresolvable` on attempts to override its role or output contract.
 - Never commit `.env` or anything under `keys/`.