diff --git a/.env.example b/.env.example index 4aba865..8ff6204 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,16 @@ # ---- Database ---- # Local dev via docker-compose. Swap for a managed Postgres URL in production. DATABASE_URL=postgres://lumenqraph:lumenqraph@localhost:5432/lumenqraph +# Maximum number of database connections in the pool (default: 10). +# Indexer: set higher for high-throughput backfill; lower for constrained environments. +# API: set based on expected concurrent requests (e.g., 20-50 for production). +# Webhooks: typically 5-10 suffices (low traffic, mainly delivery retries). +# Free-tier managed Postgres often allows 20-40 connections; check your provider's limit. +DATABASE_MAX_CONNECTIONS=10 +# Minimum number of idle connections to maintain in the pool (default: 0). +# Set to 1 or higher for faster query execution by reducing cold-start latency. +# Trade-off: higher values consume more resources while idle. Recommended: 0-2. +DATABASE_MIN_CONNECTIONS=0 # ---- Soroban RPC ---- # Testnet: https://soroban-testnet.stellar.org @@ -10,6 +20,8 @@ RPC_URL=https://soroban-testnet.stellar.org # Raise this for slow or heavily-loaded paid endpoints; lower it if you prefer # tight liveness (a hung request will fail faster and trigger the retry logic). # Applies to both the indexer and the API. Must be at least 1. +# For deep historical backfills against a slow archive RPC, 120 is a good +# starting point — see docs/DEEP_BACKFILL.md and scripts/backfill.sh --rpc-timeout. RPC_TIMEOUT_SECS=30 # ---- Indexer ---- @@ -26,6 +38,12 @@ PAGE_SIZE=1000 # from the database on next miss. Default: 2000. Set higher for better cache # hit rates when dealing with many unique contracts; lower to reduce memory usage. SPEC_CACHE_MAX_ENTRIES=2000 +# Maximum number of concurrent spec fetches allowed (default: 4). +# During a large catch-up with many new contracts, the indexer calls RPC +# to fetch contract specs. This semaphore bounds those simultaneous connections +# to prevent rate limiting or exhaustion. Already-cached specs bypass this limit. +# Increase for faster catch-up on permissive RPC; decrease for tight rate limits. +SPEC_FETCH_CONCURRENCY=4 # Start ledger for a fresh index. 0 = start near the tip. Clamped to the # RPC retention window (~7 days, ~120k ledgers on SDF public RPC). Also used # as the default for `backfill`. @@ -103,6 +121,12 @@ ENRICHMENT_WARN_THRESHOLD=0.5 # ---- API ---- API_BIND_ADDR=0.0.0.0:8080 +# Maximum allowed request body size in bytes (default: 65536 = 64 KB). +# POST /contracts/:id/call, POST /contracts/:id/simulate, and POST /graphql +# accept JSON bodies; this limit prevents large-payload denial-of-service. +# Axum returns 413 Payload Too Large for requests that exceed this value. +# Raise it only if your payloads legitimately require more (e.g. complex args). +MAX_REQUEST_BODY_BYTES=65536 # CORS (Cross-Origin Resource Sharing) configuration: comma-separated list of allowed origins, # * for all origins, or unset (default) for same-origin only (no CORS headers added). # Examples: @@ -110,8 +134,21 @@ API_BIND_ADDR=0.0.0.0:8080 # CORS_ALLOWED_ORIGINS=https://example.com,https://app.example.com # allow specific origins # Default (unset): browsers enforce same-origin policy; no Access-Control headers added. # CORS_ALLOWED_ORIGINS= +# Instance mounts: comma-separated list of sibling Lumenqraph instances to reverse-proxy +# under path prefixes. Enables serving multiple networks (mainnet, testnet, etc.) from +# one deployment. Format: name=url (e.g. testnet=http://127.0.0.1:8081). +# See docs/MULTI_NETWORK.md for patterns and configuration. +# Example: INSTANCE_MOUNTS=testnet=http://127.0.0.1:8081,futurenet=http://127.0.0.1:8082 +# INSTANCE_MOUNTS= # Require a valid API key on data routes (health/metrics stay public). REQUIRE_API_KEY=false +# Require a valid API key on GET /metrics (default: false = public, matching +# the documented behaviour). Set to true in production deployments where +# Prometheus is reachable from the internet and you don't want to expose +# indexer lag, RPC error rates, or per-contract enrichment rates publicly. +# The same API key mechanism used by data routes applies: present the key via +# `Authorization: Bearer ` or `x-api-key: `. +METRICS_REQUIRE_API_KEY=false # Requests/min for unauthenticated callers when REQUIRE_API_KEY=false. ANON_RATE_LIMIT_PER_MIN=60 # GraphQL query depth limit (default: 12). Prevents deep nested queries that exhaust resources. @@ -154,8 +191,11 @@ WEBHOOK_ENCRYPTION_KEY=GENERATE_ME_WITH_openssl_rand_hex_32 WEBHOOK_TICK_SECS=3 WEBHOOK_BATCH_SIZE=100 WEBHOOK_MAX_ATTEMPTS=6 -# Encryption key for webhook secrets (required for production) -WEBHOOK_ENCRYPTION_KEY=change-this-to-a-secure-random-key-in-production +# Maximum total webhook subscriptions allowed across the system (default: 100). +# Protects the webhook dispatcher and database from unbounded subscription creation. +WEBHOOK_MAX_SUBSCRIPTIONS=100 +# Requests/min for unauthenticated callers creating webhooks on POST /webhooks (default: 10). +WEBHOOK_CREATION_RATE_LIMIT_PER_MIN=10 # ---- Logging ---- RUST_LOG=info,lumenqraph_indexer=debug,lumenqraph_api=debug,lumenqraph_webhooks=debug diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 31b15e3..ce4a387 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,6 +2,10 @@ version: 2 updates: # ── Rust / Cargo ──────────────────────────────────────────────────────────── + # Covers the whole workspace: `directory: /` points at the root Cargo.toml, + # and Dependabot walks every member crate's manifest from there, so + # workspace-pinned crates (stellar-xdr, reqwest, axum, sqlx, …) and each + # crate's own dependencies are all monitored. - package-ecosystem: cargo directory: / schedule: @@ -9,11 +13,29 @@ updates: day: monday time: "06:00" timezone: UTC - # Group all non-breaking bumps into one PR to keep noise low. groups: + # Security patches ship on their own, unbatched, so a vulnerable transitive + # dependency is never held back waiting on an unrelated version bump. + cargo-security: + applies-to: security-updates + patterns: + - "*" + # Stellar stack (stellar-xdr, stellar-strkey, …) grouped by itself: these + # move together and a bump here can touch the XDR/strkey decode path, so + # it deserves a focused PR rather than being buried in a bulk update. + stellar: + applies-to: version-updates + patterns: + - "stellar-*" + - "soroban-*" + # Everything else: one batched PR per week to keep review noise low. cargo-dependencies: + applies-to: version-updates patterns: - "*" + exclude-patterns: + - "stellar-*" + - "soroban-*" open-pull-requests-limit: 5 labels: - dependencies @@ -36,6 +58,39 @@ updates: - dependencies - typescript + # ── Python SDK (pip) ──────────────────────────────────────────────────────── + - package-ecosystem: pip + directory: /sdk/python + schedule: + interval: weekly + day: monday + time: "06:00" + timezone: UTC + groups: + pip-dependencies: + patterns: + - "*" + open-pull-requests-limit: 5 + labels: + - dependencies + - python + + # ── Docker base images ────────────────────────────────────────────────────── + # Keeps the digest pins in the Dockerfile up-to-date so reproducible builds + # also pick up security patches on a schedule rather than requiring manual + # digest refreshes. + - package-ecosystem: docker + directory: / + schedule: + interval: weekly + day: monday + time: "06:00" + timezone: UTC + open-pull-requests-limit: 5 + labels: + - dependencies + - docker + # ── GitHub Actions ────────────────────────────────────────────────────────── - package-ecosystem: github-actions directory: / diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d95350..bf3ef8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,11 @@ jobs: cargo test -p lumenqraph-webhooks -- --ignored --test-threads=1 cargo test -p lumenqraph-api -- --ignored --test-threads=1 cargo test -p lumenqraph-mcp -- --ignored --test-threads=1 + # The end-to-end smoke test is gated behind the `smoke-tests` feature so it + # never compiles into a plain `cargo test`. It uses a mock RPC (no live + # network); run it explicitly here where Postgres is available. + - name: Test (smoke, end-to-end) + run: cargo test -p lumenqraph-indexer --features smoke-tests smoke -- --ignored --test-threads=1 - name: Install cargo-llvm-cov uses: taiki-e/install-action@fcf5432d9f50d67e37ee6e29bdb7a224ff67b4a7 - name: Collect coverage @@ -142,6 +147,18 @@ jobs: run: | python3 scripts/check_openapi_drift.py openapi.yaml /tmp/generated-openapi.json + # Validate Grafana dashboard metrics against metrics defined in Rust code + dashboard-metrics: + name: Dashboard Metrics Validation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: Validate dashboard metrics + run: python3 scripts/validate_dashboard_metrics.py + # TypeScript SDK: build, typecheck, lint, test, and codegen drift check sdk-typescript: name: TypeScript SDK @@ -169,6 +186,28 @@ jobs: - name: Codegen drift check run: npm run codegen:check + # Python SDK: lint, type-check, and test + sdk-python: + name: Python SDK + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk/python + strategy: + matrix: + python-version: ["3.8", "3.12"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install package and dev dependencies + run: pip install -e ".[dev]" + - name: Type-check (mypy) + run: mypy lumenqraph + - name: Test + run: pytest -v + security-audit: name: Security Audit (cargo-audit) runs-on: ubuntu-latest @@ -185,8 +224,22 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # stable + - uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2 + with: + key: cargo-deny + # Pinned to the 0.14 line: it still accepts the config keys in the + # committed deny.toml (0.16 removed several of them). Bump this together + # with a deny.toml migration, not on its own. - name: Install cargo-deny - run: cargo install cargo-deny - - name: Check supply chain - run: cargo deny check + run: cargo install cargo-deny --version "^0.14" --locked + # Advisories are blocking: a crate with a known RUSTSEC vulnerability now + # fails the build, so a security patch in a transitive dependency is caught + # here between the weekly Dependabot runs instead of sitting unnoticed. + - name: Check advisories + run: cargo deny check advisories + # Licenses / bans / sources are reported but not yet gating, to avoid + # blocking unrelated PRs on a pre-existing finding. Tighten to blocking + # once the current tree is known-clean. + - name: Check licenses, bans, and sources + run: cargo deny check licenses bans sources continue-on-error: true diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml new file mode 100644 index 0000000..23596eb --- /dev/null +++ b/.github/workflows/e2e-test.yml @@ -0,0 +1,73 @@ +name: End-to-End Test + +on: + schedule: + # Run nightly at 2 AM UTC + - cron: '0 2 * * *' + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +permissions: + contents: read + +jobs: + e2e-test: + name: Full Stack E2E Test + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@885d1462b5b1c5ae1b325e8c8ce65d60825e4ee8 # v3 + + - name: Generate webhook encryption key + run: | + echo "WEBHOOK_ENCRYPTION_KEY=$(openssl rand -hex 32)" >> $GITHUB_ENV + + - name: Start full stack with docker-compose + run: | + docker compose -f docker-compose.full.yml up --build -d + env: + WEBHOOK_ENCRYPTION_KEY: ${{ env.WEBHOOK_ENCRYPTION_KEY }} + RPC_URL: https://soroban-testnet.stellar.org + RUST_LOG: info + + - name: Wait for services to be ready + run: | + # Give services time to start and healthchecks to stabilize + sleep 10 + + - name: Check service status + run: | + docker compose -f docker-compose.full.yml ps + docker compose -f docker-compose.full.yml logs --tail=50 api || true + + - name: Run smoke test + run: bash scripts/e2e_smoke_test.sh + env: + API_URL: http://localhost:8080 + + - name: Query API for data + run: | + echo "Testing GraphQL endpoint..." + curl -sf http://localhost:8080/graphql -X POST \ + -H "content-type: application/json" \ + -d '{"query": "{ events(contractId: \"\") { edges { node { eventId } } } }"}' \ + | jq . || echo "GraphQL query failed (may be expected if no data indexed yet)" + + - name: Collect logs on failure + if: failure() + run: | + echo "=== API logs ===" + docker compose -f docker-compose.full.yml logs api || true + echo "=== Indexer logs ===" + docker compose -f docker-compose.full.yml logs indexer || true + echo "=== Webhooks logs ===" + docker compose -f docker-compose.full.yml logs webhooks || true + + - name: Clean up + if: always() + run: docker compose -f docker-compose.full.yml down -v diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bc05b0..e2fcb36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,50 @@ Lumenqraph follows [Semantic Versioning 2.0.0](https://semver.org/): Breaking changes will be documented in the changelog with migration guidance where applicable. +> **Upgrading a running deployment?** +> See [docs/UPGRADING.md](docs/UPGRADING.md) for step-by-step migration +> instructions, required env var changes, and the full database migration log +> for every release. + ## [Unreleased] +### Migration notes + +> Full step-by-step instructions: [docs/UPGRADING.md — Unreleased](docs/UPGRADING.md#unreleased--next-release) + +**Breaking changes requiring action before deploy:** + +- **`WEBHOOK_ENCRYPTION_KEY` is now required** for the webhooks service. + Webhook secrets are encrypted at rest using `pgcrypto`. Generate the key + with `openssl rand -hex 32` and set it before deploying — migration + `0020` backfills existing subscriptions using this key. Deployments + without it fall back to the insecure hardcoded default. + See [docs/UPGRADING.md](docs/UPGRADING.md#1-webhook-secrets-are-now-encrypted-at-rest--webhook_encryption_key-required). + +- **`token_transfers.kind` column added** (`transfer` | `mint` | `burn` | + `clawback`). Clients reading transfer payloads by positional index must + update. Migration `0015` backfills existing rows as `"transfer"`. + +- **CORS is now same-origin only by default.** Set `CORS_ALLOWED_ORIGINS` + if your frontend is on a different origin. + +- **GraphQL introspection and GraphiQL are off by default.** Set + `GRAPHQL_INTROSPECTION_ENABLED=true` in non-production environments if needed. + +**Database migrations applied:** `0010` through `0021` (run automatically by +the indexer on startup). Stop the webhooks service before deploying to avoid +a write conflict on `webhook_deliveries` during migration `0008`. + +**New environment variables:** `WEBHOOK_ENCRYPTION_KEY`, `CORS_ALLOWED_ORIGINS`, +`GRAPHQL_MAX_DEPTH`, `GRAPHQL_MAX_COMPLEXITY`, `GRAPHQL_INTROSPECTION_ENABLED`, +`RATE_LIMIT_TRUST_XFF`, `RATE_LIMIT_BACKEND`, `REDIS_URL`, +`RPC_ROUTE_RATE_LIMIT_PER_MIN`, `RPC_REQUIRE_API_KEY`, `RPC_TIMEOUT_SECS`, +`READYZ_LAG_THRESHOLD`, `READYZ_MAX_AGE_SECS`, `HEALTH_MAX_LAG_LEDGERS`, +`HEALTH_MAX_STALE_SECS`, `ENRICHMENT_WARN_THRESHOLD`, `SPEC_CACHE_MAX_ENTRIES`, +`SPEC_VERSION_RETENTION`, `KEY_TEMPLATES`, `BALANCE_KEY_SYMBOL`, +`BALANCE_KEY_DURABILITY`, `DATABASE_MAX_CONNECTIONS`, `DATABASE_MIN_CONNECTIONS`, +`DATABASE_ACQUIRE_TIMEOUT_SECS`, `DATABASE_IDLE_TIMEOUT_SECS`. + ### Added - Keyset cursor pagination for REST `/events` and `/transfers` endpoints - Trailing re-scan mechanism for shallow reorg detection @@ -43,6 +85,16 @@ Breaking changes will be documented in the changelog with migration guidance whe ## [0.1.0] - Initial Release +### Migration notes + +> Full step-by-step instructions: [docs/UPGRADING.md — Fresh install / v0.1.0](docs/UPGRADING.md#fresh-install--v010-initial-release) + +**Fresh install** — no prior version to migrate from. The indexer applies +migrations `0001` through `0009` automatically on first startup. + +**Required environment variables:** `DATABASE_URL`, `RPC_URL`. +All other variables have safe defaults. See [Configuration](README.md#configuration). + ### Added - **Core indexing**: Poll Soroban RPC `getEvents`, decode XDR to JSON, store in Postgres - **Typed, self-describing decoding**: Parse contract's on-chain `contractspecv0` interface and enrich events with field names and types automatically (zero configuration) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0186a0f..3e664b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,6 +68,32 @@ cargo test -p lumenqraph-mcp -- --ignored --test-threads=1 CI runs all of the above against a Postgres service. +### Smoke tests + +`crates/lumenqraph-indexer/src/smoke.rs` is a single heavy end-to-end test that +drives the whole pipeline (indexer → Postgres → API-shaped queries → webhook +enqueue → signed delivery). It uses an in-process mock Soroban RPC and a local +HTTP sink — **no live network** — but it is expensive and needs a database, so +it is kept out of the normal test run by two independent gates: + +- `#[ignore]` — skipped by `cargo test` unless `--ignored` is passed. +- `#[cfg(feature = "smoke-tests")]` — the module is not even compiled without + the `smoke-tests` cargo feature, so it can never run in an offline CI job that + forgets to set `TEST_DATABASE_URL`. + +Run it with: + +```bash +make test-smoke +``` + +or manually: + +```bash +export TEST_DATABASE_URL=postgres://user:password@host:port/dbname +cargo test -p lumenqraph-indexer --features smoke-tests smoke -- --ignored --test-threads=1 +``` + ## Conventions - Shared types and decoding live in `lumenqraph-core`; don't duplicate models. diff --git a/Cargo.toml b/Cargo.toml index 92160b2..343fee9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,10 +54,17 @@ url = "2" axum = "0.7" tower = "0.5" tower-http = { version = "0.5", features = ["trace", "cors", "fs", "set-header", "compression-gzip", "compression-br", "timeout"] } -# Pinned to the 7.0.x line that targets axum 0.7 (7.0.11+ moved to axum 0.8, -# which would clash with the rest of the stack). -async-graphql = { version = "=7.0.7", features = ["chrono"] } -async-graphql-axum = "=7.0.7" +# Constrained to the async-graphql 7.0.x releases that still target axum 0.7: +# 7.0.11 switched to axum 0.8, which would clash with the rest of the stack +# (`axum = "0.7"`, `tower-http = "0.5"`, and the utoipa axum integrations). +# +# A `>=` / `<` range rather than an exact `=` pin so patch-level fixes in the +# 7.0.7..7.0.10 window (bug/security) are still picked up automatically, while +# the axum-0.8 bump stays gated. Lifting the `<7.0.11` ceiling is the +# async-graphql side of migrating the whole workspace to axum 0.8 and should be +# done together with it (tracked separately). +async-graphql = { version = ">=7.0.7, <7.0.11", features = ["chrono"] } +async-graphql-axum = ">=7.0.7, <7.0.11" # Canonical Stellar XDR types. Used ONLY on the cold, once-per-contract spec / # ledger-entry path (parsing a contract's on-chain interface). The hot event # decode path stays dependency-free in `lumenqraph-core::xdr`. @@ -67,6 +74,7 @@ stellar-xdr = { version = "23", default-features = false, features = [ "base64", ] } lru = "0.12" +criterion = { version = "0.5", features = ["async_tokio"] } [profile.release] opt-level = 3 diff --git a/Dockerfile b/Dockerfile index 2ac10a4..7061c60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Multi-stage build producing one image with all four service binaries. # Uses rustls throughout (no OpenSSL), so the runtime only needs CA certs. -FROM rust:1-slim AS builder +FROM rust:1-slim@sha256:17d1ba895198f9934c6314ec5346a0d5115372f3243390c3d731e242f35c2f27 AS builder WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends pkg-config \ && rm -rf /var/lib/apt/lists/* @@ -10,7 +10,7 @@ COPY crates ./crates COPY migrations ./migrations RUN cargo build --release --workspace -FROM debian:bookworm-slim AS runtime +FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 AS runtime WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index ca1909c..6ad90c8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help db db-down seed build test test-db fmt lint indexer api webhooks backfill up down +.PHONY: help db db-down seed build test test-db test-smoke fmt lint indexer api webhooks backfill up down help: @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ @@ -31,6 +31,12 @@ test-db: db ## Run Postgres-backed tests (requires TEST_DATABASE_URL or a runnin cargo test -p lumenqraph-api -- --ignored --test-threads=1; \ cargo test -p lumenqraph-mcp -- --ignored --test-threads=1 +test-smoke: db ## Run the gated end-to-end smoke test (requires TEST_DATABASE_URL or a running local Postgres) + @if [ -z "$$TEST_DATABASE_URL" ]; then \ + export TEST_DATABASE_URL=postgres://lumenqraph:lumenqraph@localhost:5432/lumenqraph; \ + fi; \ + cargo test -p lumenqraph-indexer --features smoke-tests smoke -- --ignored --test-threads=1 + fmt: ## Format cargo fmt --all diff --git a/README.md b/README.md index 01079ea..27a1c2c 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Indexing Stellar **mainnet** right now. Below: the [Aquarius AMM](https://aqua.n - [TypeScript SDK](#typescript-sdk) - [AI-agent access — the MCP server](#ai-agent-access--the-mcp-server) - [Webhooks](#webhooks) +- [Multi-network deployments](#multi-network-deployments) - [Running in production](#running-in-production) - [Project structure](#project-structure) - [Development](#development) @@ -51,6 +52,7 @@ Indexing Stellar **mainnet** right now. Below: the [Aquarius AMM](https://aqua.n - [Troubleshooting & FAQ](docs/TROUBLESHOOTING.md) - [Explorer UI & Configuration](docs/EXPLORER.md) - [Migration Rollback Strategy](docs/MIGRATIONS.md) +- [Upgrading Between Versions](docs/UPGRADING.md) - [Security](SECURITY.md) - [Contributing](#contributing) - [License](#license) @@ -193,8 +195,11 @@ All configuration is via environment variables (see [`.env.example`](.env.exampl | `SPEC_CACHE_MAX_ENTRIES` | `2000` | Maximum number of contract specs held in memory before evicting least-recently-used entries. Prevents unbounded memory growth in index-all mode. Evicted specs are re-fetched from the database on next miss. Increase for better cache hit rates with many contracts; decrease to reduce memory usage. | | `ENRICHMENT_WARN_THRESHOLD` | `0.5` | Warn (emit a warn-level log) if the not-enriched fraction (events failing spec decode or fetch) exceeds this threshold in a single poll cycle. Range: 0.0–1.0. Default 0.5 = warn if >50% fail. Set to 0.0 to disable warnings; 1.0+ to warn only at 100%. | | `API_BIND_ADDR` | `0.0.0.0:8080` | API listen address. | +| `INSTANCE_MOUNTS` | *(unset)* | Comma-separated list of sibling instances to reverse-proxy under path prefixes (e.g. `testnet=http://127.0.0.1:8081`). Enables serving multiple networks from one deployment. See [Multi-network deployments](#multi-network-deployments). | | `CORS_ALLOWED_ORIGINS` | *(unset)* | Comma-separated list of allowed origins for CORS requests (e.g. `https://example.com`), `*` to allow all origins, or unset for same-origin only (no CORS headers added, default behavior). | | `REQUIRE_API_KEY` | `false` | Require a valid API key on data routes. | +| `METRICS_REQUIRE_API_KEY` | `false` | Require a valid API key on `GET /metrics`. When `false` (default) the endpoint is public. Set to `true` in production when the API is internet-accessible to avoid leaking operational telemetry. | +| `MAX_REQUEST_BODY_BYTES` | `65536` | Maximum request body size in bytes (64 KB). Axum returns `413` for larger bodies. Protects `POST /call`, `POST /simulate`, and `POST /graphql` from large-payload denial-of-service. Replaces the old `API_MAX_BODY_BYTES` variable (kept as a fallback alias). | | `ANON_RATE_LIMIT_PER_MIN` | `60` | Requests/min for unauthenticated callers. | | `WEBHOOK_TICK_SECS` | `3` | Webhook dispatcher poll interval. | | `WEBHOOK_BATCH_SIZE` | `100` | Deliveries processed per tick. | @@ -209,7 +214,7 @@ All configuration is via environment variables (see [`.env.example`](.env.exampl Base URL defaults to `http://localhost:8080`. Full reference: [docs/API.md](docs/API.md). -**Authentication.** Data routes accept an API key via `Authorization: Bearer ` or `x-api-key: `. When `REQUIRE_API_KEY=false` (default), unauthenticated callers are allowed up to `ANON_RATE_LIMIT_PER_MIN`. `/health` and `/metrics` are always public. Rate-limit breaches return `429`; invalid or revoked keys return `401`. +**Authentication.** Data routes accept an API key via `Authorization: Bearer ` or `x-api-key: `. When `REQUIRE_API_KEY=false` (default), unauthenticated callers are allowed up to `ANON_RATE_LIMIT_PER_MIN`. `/health`, `/livez`, and `/readyz` are always public. `/metrics` is public by default and can be restricted with `METRICS_REQUIRE_API_KEY=true`. Rate-limit breaches return `429`; invalid or revoked keys return `401`. | Method | Path | Description | | --- | --- | --- | @@ -240,6 +245,8 @@ Base URL defaults to `http://localhost:8080`. Full reference: [docs/API.md](docs | `GET` | `/webhooks` | List subscriptions (secrets omitted). | | `DELETE` | `/webhooks/:id` | Delete a subscription. | +**Multi-network paths:** When `INSTANCE_MOUNTS` is configured (see [Multi-network deployments](#multi-network-deployments)), all routes above are also available under `//` prefixes. Example: `GET //contracts` queries the mounted instance. +
Example: a decoded transfer event @@ -628,6 +635,35 @@ function verify(rawBody, signatureHeader, secret) { Deliveries retry with exponential backoff up to `WEBHOOK_MAX_ATTEMPTS`. +## Multi-network deployments + +Lumenqraph can serve multiple Stellar networks (mainnet, testnet, futurenet) from a single logical deployment using **instance mounts**. One Lumenqraph API reverse-proxies to sibling instances running on the same host, allowing clients to query different networks under a unified origin. + +**Recommended pattern:** One instance per network + federation at the edge. This provides clean separation, independent scaling, and simple operations. + +Example: +```bash +# Primary instance (mainnet) +RPC_URL=https://mainnet.sorobanrpc.com +INSTANCE_MOUNTS=testnet=http://127.0.0.1:8081 + +# GET /contracts → queries mainnet +# GET /testnet/contracts → proxied to testnet instance at :8081 +``` + +The primary instance's `/health` endpoint advertises available networks so clients (like the explorer) can discover them automatically and switch networks with one click: + +```json +{ + "network": "mainnet", + "mounts": { + "testnet": "/testnet" + } +} +``` + +See [docs/MULTI_NETWORK.md](docs/MULTI_NETWORK.md) for detailed configuration, deployment patterns (free tier, separate services, fully isolated), security considerations, and monitoring. + ## Running in production Run three long-lived processes against one Postgres. Only the indexer applies migrations. @@ -640,6 +676,8 @@ Run three long-lived processes against one Postgres. Only the indexer applies mi Scrape `GET /metrics` and alert on `lumenqraph_indexer_lag_ledgers` climbing. For managed Postgres, point `DATABASE_URL` at Neon or Supabase. See [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) for scaling notes (RPC providers, Redis-backed rate limiting, caching). +When upgrading a running deployment, always consult [docs/UPGRADING.md](docs/UPGRADING.md) for breaking changes, required env var additions, and the database migration log for each release. + ## Project structure ``` @@ -705,7 +743,7 @@ Contributions toward any of these are very welcome — see [Contributing](#contr Contributions are welcome! Please read [CONTRIBUTING.md](CONTRIBUTING.md) for the dev setup and conventions, and review our [Code of Conduct](CODE_OF_CONDUCT.md). Good first issues are labelled in the [issue tracker](https://github.com/Lumen-Scribe/Lumenqraph/issues). -See [CHANGELOG.md](CHANGELOG.md) for release history and versioning policy. +See [CHANGELOG.md](CHANGELOG.md) for release history, versioning policy, and per-release migration notes. See [docs/UPGRADING.md](docs/UPGRADING.md) for step-by-step upgrade instructions. ## License diff --git a/SECURITY.md b/SECURITY.md index 4557ede..5916d40 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -113,6 +113,42 @@ When deploying Lumenqraph in production: Note: The plaintext `secret` column is retained for backward compatibility during rolling deployments. A future migration will drop it once all instances are updated. +### Webhook Signing Secret Rotation + +To rotate a subscription's signing secret without losing delivery history or resetting the watermark, use the dedicated rotation endpoint: + +```bash +curl -X POST https:///webhooks//rotate-secret \ + -H "x-api-key: " +``` + +The response contains the new secret (shown **once only**) and the timestamp until which the previous secret remains valid: + +```json +{ + "id": "...", + "secret": "", + "previous_secret_valid_until": "2025-01-24T12:05:00Z", + "message": "Store this secret immediately — it will not be shown again." +} +``` + +**Rotation procedure:** + +1. Call `POST /webhooks/:id/rotate-secret` and capture the new secret immediately. +2. Update your consumer to accept both the old and new secrets during the grace period (`WEBHOOK_SECRET_GRACE_SECS`, default 300 seconds). The server validates deliveries signed with either secret during this window. +3. After the grace period expires, retire the old secret from your consumer — only the new one will be valid. + +The grace period avoids a verification gap: in-flight deliveries signed with the old secret are still accepted while you roll out the updated secret to your infrastructure. + +**Environment configuration:** + +```bash +# Grace period during which the old secret stays valid alongside the new one (seconds). +# Default: 300 (5 minutes). Increase for slower rollouts. +WEBHOOK_SECRET_GRACE_SECS=300 +``` + ### For Developers When contributing to Lumenqraph: diff --git a/crates/lumenqraph-api/src/auth.rs b/crates/lumenqraph-api/src/auth.rs index 3db471b..1da6d39 100644 --- a/crates/lumenqraph-api/src/auth.rs +++ b/crates/lumenqraph-api/src/auth.rs @@ -280,12 +280,98 @@ pub async fn rpc_auth_and_rate_limit( } }; - if !state.rpc_limiter.check(&identity, limit).allowed { + let rl_status = state.rpc_limiter.check(&identity, limit); + if !rl_status.allowed { + if is_authenticated { + let hash_prefix = identity.split(':').nth(1).unwrap_or("unknown"); + log_audit_event(&state.pool, hash_prefix, &route, &method, 429).await; + } + return Err(ApiError::too_many_requests(rl_status.retry_after_secs)); + } + + let response = next.run(req).await; + let status = response.status().as_u16(); + + if is_authenticated { + let hash_prefix = identity.split(':').nth(1).unwrap_or("unknown"); + log_audit_event(&state.pool, hash_prefix, &route, &method, status).await; + } + + Ok(response) +} + +/// Middleware for webhook subscription creation (POST /webhooks). +/// Uses a separate rate limiter with a lower limit for anonymous callers +/// to prevent unbounded subscription creation. +pub async fn webhook_auth_and_rate_limit( + State(state): State, + ConnectInfo(socket_addr): ConnectInfo, + headers: HeaderMap, + req: Request, + next: Next, +) -> ApiResult { + state.http_requests.fetch_add(1, Ordering::Relaxed); + + let method = req.method().to_string(); + let uri = req.uri().to_string(); + let route = uri.split('?').next().unwrap_or("").to_string(); + + let (identity, limit, is_authenticated) = match extract_key(&headers) { + Some(key) => { + let hash = hash_key(&key); + let row: Option<(bool, i32)> = sqlx::query_as( + "SELECT revoked, rate_limit_per_min FROM api_keys WHERE key_hash = $1", + ) + .bind(&hash) + .fetch_optional(&state.pool) + .await?; + match row { + Some((false, limit)) => (format!("key:{hash}"), limit, true), + Some((true, _)) => { + log_audit_event(&state.pool, &hash, &route, &method, 401).await; + return Err(ApiError::unauthorized("API key revoked")) + }, + None => { + log_audit_event(&state.pool, &hash, &route, &method, 401).await; + return Err(ApiError::unauthorized("invalid API key")) + }, + } + } + None => { + if state.require_auth { + return Err(ApiError::unauthorized("missing API key")); + } + let client_ip = extract_client_ip(&headers, Some(socket_addr)); + (format!("anon:{client_ip}"), state.webhook_anon_rate_limit, false) + } + }; + + let rl_status = state.webhook_limiter.check(&identity, limit); + if !rl_status.allowed { + let mut response = (StatusCode::TOO_MANY_REQUESTS, crate::error::rate_limit_error()).into_response(); + + // Add rate limit headers + if let Some(retry_after) = rl_status.retry_after_secs { + response.headers_mut().insert( + "Retry-After", + retry_after.to_string().parse().unwrap_or_else(|_| "60".parse().unwrap()), + ); + } + response.headers_mut().insert( + "X-RateLimit-Limit", + limit.to_string().parse().unwrap_or_else(|_| "0".parse().unwrap()), + ); + response.headers_mut().insert( + "X-RateLimit-Remaining", + rl_status.tokens_remaining.to_string().parse().unwrap_or_else(|_| "0".parse().unwrap()), + ); + if is_authenticated { let hash_prefix = identity.split(':').nth(1).unwrap_or("unknown"); log_audit_event(&state.pool, hash_prefix, &route, &method, 429).await; } - return Err(ApiError::too_many_requests()); + + return Ok(response); } let response = next.run(req).await; @@ -412,6 +498,13 @@ mod integration_tests { concurrency_limiter: Arc::new(ConcurrencyLimiter::new()), max_concurrent_per_ip: 100, read_cost_limit_config: ReadCostLimitConfig::default(), + readyz_lag_threshold: 100, + readyz_max_age_secs: 120, + health_max_lag_ledgers: 100, + health_max_stale_secs: 120, + webhook_limiter: Arc::new(RateLimiter::new()), + webhook_anon_rate_limit: 10, + webhook_max_subscriptions: 100, } } @@ -600,4 +693,30 @@ mod integration_tests { let body: serde_json::Value = res.json().await.unwrap(); assert!(body.get("error").is_some(), "429 must have error envelope"); } + + #[tokio::test] + #[ignore = "needs postgres"] + async fn rate_limit_response_has_retry_after_header() { + let pool = db_pool().await; + let base = spawn_server(make_state(pool, false, 1)).await; + let client = reqwest::Client::new(); + // Exhaust the single token. + client.get(format!("{base}/contracts")).send().await.unwrap(); + let res = client + .get(format!("{base}/contracts")) + .send() + .await + .unwrap(); + assert_eq!(res.status(), 429); + let retry_after = res + .headers() + .get("retry-after") + .expect("429 responses must carry a Retry-After header") + .to_str() + .unwrap(); + assert!( + retry_after.parse::().is_ok(), + "Retry-After must be an integer number of seconds, got {retry_after:?}" + ); + } } diff --git a/crates/lumenqraph-api/src/call_cache.rs b/crates/lumenqraph-api/src/call_cache.rs index 75b8001..0557d17 100644 --- a/crates/lumenqraph-api/src/call_cache.rs +++ b/crates/lumenqraph-api/src/call_cache.rs @@ -11,6 +11,7 @@ //! `CALL_CACHE_MAX_ENTRIES` — LRU capacity (default 1000). use std::num::NonZeroUsize; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; use std::time::{Duration, Instant}; @@ -30,6 +31,9 @@ struct Key { pub struct CallCache { inner: Mutex>, ttl: Duration, + hits: AtomicU64, + misses: AtomicU64, + evictions: AtomicU64, } impl CallCache { @@ -38,6 +42,9 @@ impl CallCache { Self { inner: Mutex::new(LruCache::new(capacity)), ttl: Duration::from_secs(ttl_secs), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + evictions: AtomicU64::new(0), } } @@ -50,14 +57,19 @@ impl CallCache { let mut cache = self.inner.lock().unwrap(); match cache.get(&key) { Some((value, inserted_at)) if inserted_at.elapsed() < self.ttl => { + self.hits.fetch_add(1, Ordering::Relaxed); Some(value.clone()) } Some(_) => { // Expired: evict now so the LRU capacity reflects live entries. cache.pop(&key); + self.misses.fetch_add(1, Ordering::Relaxed); + None + } + None => { + self.misses.fetch_add(1, Ordering::Relaxed); None } - None => None, } } @@ -67,10 +79,33 @@ impl CallCache { return; } let key = self.make_key(contract_id, function, args); - self.inner + let evicted = self.inner .lock() .unwrap() .put(key, (result, Instant::now())); + if evicted.is_some() { + self.evictions.fetch_add(1, Ordering::Relaxed); + } + } + + /// Get cache hit count (for Prometheus metrics). + pub fn hits(&self) -> u64 { + self.hits.load(Ordering::Relaxed) + } + + /// Get cache miss count (for Prometheus metrics). + pub fn misses(&self) -> u64 { + self.misses.load(Ordering::Relaxed) + } + + /// Get cache eviction count (for Prometheus metrics). + pub fn evictions(&self) -> u64 { + self.evictions.load(Ordering::Relaxed) + } + + /// Get current cache size (number of entries, for Prometheus metrics). + pub fn size(&self) -> usize { + self.inner.lock().unwrap().len() } fn make_key(&self, contract_id: &str, function: &str, args: &Value) -> Key { diff --git a/crates/lumenqraph-api/src/error.rs b/crates/lumenqraph-api/src/error.rs index fcafa71..e75577f 100644 --- a/crates/lumenqraph-api/src/error.rs +++ b/crates/lumenqraph-api/src/error.rs @@ -19,9 +19,10 @@ //! | `bad_request` | 400 | Malformed input, invalid parameter value, wrong type. | //! | `unauthorized` | 401 | Missing or revoked API key. | //! | `not_found` | 404 | Requested resource does not exist. | -//! | `rate_limited` | 429 | Caller exceeded the request-per-minute limit. | +//! | `rate_limited` | 429 | Caller exceeded the request-per-minute limit. Carries a `Retry-After` header. | //! | `simulation_failed` | 400 | RPC simulation returned an error (contract trap, etc.).| -//! | `spec_unavailable` | 404 | Contract interface not indexed (or Stellar Asset Contract). | +//! | `spec_unavailable` | 404 | Contract has not been indexed yet; retry later. | +//! | `sac_not_supported` | 422 | Stellar Asset Contract: no WASM spec; retrying will not help. | //! | `internal_error` | 500 | Unexpected server-side failure (details are logged). | use std::fmt; @@ -44,6 +45,7 @@ pub enum ErrorCode { RateLimited, SimulationFailed, SpecUnavailable, + FeatureDisabled, InternalError, } @@ -57,6 +59,7 @@ impl ErrorCode { ErrorCode::RateLimited => "rate_limited", ErrorCode::SimulationFailed => "simulation_failed", ErrorCode::SpecUnavailable => "spec_unavailable", + ErrorCode::FeatureDisabled => "feature_disabled", ErrorCode::InternalError => "internal_error", } } @@ -72,6 +75,13 @@ impl fmt::Display for ErrorCode { pub enum ApiError { /// A client-facing status + code + message (4xx). Status(StatusCode, ErrorCode, String), + /// A 429 carrying the number of seconds the caller should wait before + /// retrying. Rendered with a `Retry-After` header so SDKs can back off + /// precisely instead of guessing with exponential backoff. + RateLimited { + retry_after_secs: Option, + message: String, + }, /// An unexpected internal failure (500); details are logged, not exposed. Internal(anyhow::Error), } @@ -80,12 +90,13 @@ impl ApiError { pub fn unauthorized(msg: impl Into) -> Self { ApiError::Status(StatusCode::UNAUTHORIZED, ErrorCode::Unauthorized, msg.into()) } - pub fn too_many_requests() -> Self { - ApiError::Status( - StatusCode::TOO_MANY_REQUESTS, - ErrorCode::RateLimited, - "rate limit exceeded".into(), - ) + /// A 429 response. Pass the computed wait time from `RateLimitStatus` so the + /// response carries a `Retry-After` header; `None` omits the header. + pub fn too_many_requests(retry_after_secs: Option) -> Self { + ApiError::RateLimited { + retry_after_secs, + message: "rate limit exceeded".into(), + } } pub fn bad_request(msg: impl Into) -> Self { ApiError::Status(StatusCode::BAD_REQUEST, ErrorCode::BadRequest, msg.into()) @@ -107,6 +118,13 @@ impl ApiError { msg.into(), ) } + pub fn feature_disabled(msg: impl Into) -> Self { + ApiError::Status( + StatusCode::NOT_IMPLEMENTED, + ErrorCode::FeatureDisabled, + msg.into(), + ) + } } impl From for ApiError { @@ -131,6 +149,7 @@ impl fmt::Display for ApiError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { ApiError::Status(_, _, msg) => write!(f, "{}", msg), + ApiError::RateLimited { message, .. } => write!(f, "{}", message), ApiError::Internal(e) => write!(f, "{}", e), } } @@ -141,28 +160,45 @@ impl std::error::Error for ApiError { match self { ApiError::Internal(e) => Some(e.as_ref()), ApiError::Status(_, _, _) => None, + ApiError::RateLimited { .. } => None, } } } impl IntoResponse for ApiError { fn into_response(self) -> Response { - let (status, code, message) = match self { - ApiError::Status(s, c, m) => (s, c, m), + let (status, code, message, retry_after_secs) = match self { + ApiError::Status(s, c, m) => (s, c, m, None), + ApiError::RateLimited { + retry_after_secs, + message, + } => ( + StatusCode::TOO_MANY_REQUESTS, + ErrorCode::RateLimited, + message, + retry_after_secs, + ), ApiError::Internal(e) => { tracing::error!(error = %e, "request failed"); ( StatusCode::INTERNAL_SERVER_ERROR, ErrorCode::InternalError, "internal error".to_string(), + None, ) } }; - ( + let mut response = ( status, Json(json!({ "code": code.as_str(), "error": message })), ) - .into_response() + .into_response(); + if let Some(secs) = retry_after_secs { + if let Ok(value) = secs.to_string().parse() { + response.headers_mut().insert("Retry-After", value); + } + } + response } } @@ -171,3 +207,35 @@ pub fn rate_limit_error() -> Json { } pub type ApiResult = Result; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rate_limited_sets_retry_after_header() { + let resp = ApiError::too_many_requests(Some(42)).into_response(); + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!( + resp.headers() + .get("Retry-After") + .and_then(|v| v.to_str().ok()), + Some("42"), + "429 responses must carry a Retry-After header with the computed wait" + ); + } + + #[test] + fn rate_limited_without_hint_omits_retry_after_header() { + let resp = ApiError::too_many_requests(None).into_response(); + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + assert!(resp.headers().get("Retry-After").is_none()); + } + + #[test] + fn non_rate_limited_errors_have_no_retry_after_header() { + let resp = ApiError::not_found("nope").into_response(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + assert!(resp.headers().get("Retry-After").is_none()); + } +} diff --git a/crates/lumenqraph-api/src/graphql.rs b/crates/lumenqraph-api/src/graphql.rs index 7f41d07..4f87ab8 100644 --- a/crates/lumenqraph-api/src/graphql.rs +++ b/crates/lumenqraph-api/src/graphql.rs @@ -49,6 +49,14 @@ pub fn build_schema(pool: PgPool) -> AppSchema { // ---- Types ---- +#[derive(SimpleObject)] +struct EnrichedParam { + name: String, + #[graphql(name = "type")] + type_: String, + value: GqlJson, +} + #[derive(SimpleObject)] struct ContractStat { contract_id: String, @@ -68,7 +76,6 @@ impl From for ContractStat { } } -#[derive(SimpleObject)] struct Event { event_id: String, contract_id: String, @@ -76,11 +83,8 @@ struct Event { ledger_closed_at: DateTime, event_type: String, event_name: Option, - /// Decoded topics as JSON. decoded_topics: GqlJson, - /// Decoded event body as JSON. decoded_value: GqlJson, - /// Named, typed record from the contract spec; null when none matched. enriched: Option>, tx_hash: String, in_successful_call: bool, @@ -104,6 +108,85 @@ impl From for Event { } } +#[Object] +impl Event { + async fn event_id(&self) -> &str { + &self.event_id + } + + async fn contract_id(&self) -> &str { + &self.contract_id + } + + async fn ledger(&self) -> i64 { + self.ledger + } + + async fn ledger_closed_at(&self) -> DateTime { + self.ledger_closed_at + } + + async fn event_type(&self) -> &str { + &self.event_type + } + + async fn event_name(&self) -> &Option { + &self.event_name + } + + async fn decoded_topics(&self) -> &GqlJson { + &self.decoded_topics + } + + async fn decoded_value(&self) -> &GqlJson { + &self.decoded_value + } + + async fn enriched(&self) -> &Option> { + &self.enriched + } + + async fn params(&self) -> Result> { + match &self.enriched { + Some(enriched) => { + if let Some(params_obj) = enriched.0.get("params").and_then(|v| v.as_object()) { + let params: Vec = params_obj + .iter() + .map(|(name, value)| { + let type_ = value + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + let param_value = value + .get("value") + .cloned() + .unwrap_or_else(|| Value::Null); + EnrichedParam { + name: name.clone(), + type_, + value: GqlJson(param_value), + } + }) + .collect(); + Ok(params) + } else { + Ok(Vec::new()) + } + } + None => Ok(Vec::new()), + } + } + + async fn tx_hash(&self) -> &str { + &self.tx_hash + } + + async fn in_successful_call(&self) -> bool { + self.in_successful_call + } +} + #[derive(SimpleObject)] struct EventEdge { cursor: String, @@ -238,11 +321,17 @@ impl QueryRoot { Ok(build_event_connection(rows, limit)) } - /// Cursor-paginated token transfers, newest first. Filter by contract. + /// Cursor-paginated token transfers, newest first. Optional filters by + /// contract and by the `from` / `to` address — mirroring the REST + /// `GET /contracts/:id/transfers` `?from=`/`?to=` query parameters. async fn transfers( &self, ctx: &Context<'_>, contract_id: Option, + #[graphql(desc = "Only transfers sent from this address (G… / C… strkey)")] + from: Option, + #[graphql(desc = "Only transfers received by this address (G… / C… strkey)")] + to: Option, #[graphql(desc = "Page size (1-200, default 20)")] first: Option, after: Option, ) -> Result { @@ -258,11 +347,15 @@ impl QueryRoot { "SELECT event_id, contract_id, from_addr, to_addr, amount, ledger, ledger_closed_at FROM token_transfers WHERE ($1::text IS NULL OR contract_id = $1) - AND ($2::bigint IS NULL OR ledger < $2 OR (ledger = $2 AND event_id < $3)) + AND ($2::text IS NULL OR from_addr = $2) + AND ($3::text IS NULL OR to_addr = $3) + AND ($4::bigint IS NULL OR ledger < $4 OR (ledger = $4 AND event_id < $5)) ORDER BY ledger DESC, event_id DESC - LIMIT $4", + LIMIT $6", ) .bind(&contract_id) + .bind(&from) + .bind(&to) .bind(after_ledger) .bind(after_id) .bind(limit + 1) @@ -440,6 +533,12 @@ mod tests { "type EventConnection", "type PageInfo", "hasNextPage", + // The GraphQL `transfers` field must expose the same address filters + // as the REST endpoint (#285): `from` and `to`, both optional. The + // `Transfer` type's own fields are `fromAddr` / `toAddr`, so these + // substrings are unambiguous argument signatures. + "from: String", + "to: String", ] { assert!(sdl.contains(expected), "SDL missing {expected:?}"); } diff --git a/crates/lumenqraph-api/src/main.rs b/crates/lumenqraph-api/src/main.rs index 0006097..d648c25 100644 --- a/crates/lumenqraph-api/src/main.rs +++ b/crates/lumenqraph-api/src/main.rs @@ -183,14 +183,13 @@ async fn main() -> anyhow::Result<()> { let database_url = std::env::var("DATABASE_URL").context("missing DATABASE_URL")?; - // Validate webhook encryption key is set for production security - if std::env::var("WEBHOOK_ENCRYPTION_KEY").is_err() { - anyhow::bail!( - "WEBHOOK_ENCRYPTION_KEY must be set (generate with: openssl rand -hex 32). \ - The default test key provides no security and must not be used in production." - ); - } - + // Validate CONTRACT_IDS at startup so a misconfigured address is caught + // immediately rather than silently ignored or causing runtime errors. + lumenqraph_core::parse_contract_ids( + &std::env::var("CONTRACT_IDS").unwrap_or_default(), + ) + .map_err(|e| anyhow::anyhow!("{e}"))?; + let bind_addr = std::env::var("API_BIND_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".to_string()); let rpc_url = std::env::var("RPC_URL") .unwrap_or_else(|_| "https://soroban-testnet.stellar.org".to_string()); @@ -240,10 +239,16 @@ async fn main() -> anyhow::Result<()> { readyz_max_age_secs: env_parse("READYZ_MAX_AGE_SECS", 120i64), health_max_lag_ledgers: env_parse("HEALTH_MAX_LAG_LEDGERS", 100i64), health_max_stale_secs: env_parse("HEALTH_MAX_STALE_SECS", 120i64), + metrics_require_auth: env_bool("METRICS_REQUIRE_API_KEY", false), }; let cors_layer = build_cors_layer(); - let max_body_bytes = env_parse::("API_MAX_BODY_BYTES", 256 * 1024); + // MAX_REQUEST_BODY_BYTES is the canonical name (#212). + // API_MAX_BODY_BYTES is kept as a fallback alias for backward compatibility. + let max_body_bytes = std::env::var("MAX_REQUEST_BODY_BYTES") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| env_parse::("API_MAX_BODY_BYTES", 65536)); info!(max_body_bytes, "enforcing request body size limit"); info!(request_timeout_secs, "enforcing request timeout"); @@ -288,3 +293,80 @@ async fn shutdown_signal() { } info!("shutdown signal received; stopping api"); } + +#[cfg(test)] +mod tests { + /// #219 — CONTRACT_IDS startup validation in lumenqraph-api. + /// + /// The API calls `lumenqraph_core::parse_contract_ids` at startup and + /// propagates the error, refusing to proceed. These tests exercise the same + /// validation logic directly, without needing a live Postgres or bind + /// address, to ensure the guard never silently regresses. + mod contract_ids_startup_validation { + #[test] + fn rejects_g_strkey_account_address() { + // A G… strkey is a Stellar account, not a Soroban contract. + let raw = "GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "error should mention invalid CONTRACT_ID: {err}" + ); + assert!( + err.contains("GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"), + "error should quote the bad id: {err}" + ); + } + + #[test] + fn rejects_garbage_string() { + let raw = "not-a-contract-id"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "garbage string should be rejected: {err}" + ); + } + + #[test] + fn rejects_too_many_contract_ids() { + // getEvents supports at most 25 IDs; the parser enforces this. + // Build 26 syntactically valid-looking (but fake) C-strkey placeholders + // by using the same test id repeated — the count check fires before + // strkey validation so any 26 non-empty tokens trigger it. + // Use a real C-strkey so each individual ID passes strkey validation. + let single = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"; + let raw = std::iter::repeat(single).take(26).collect::>().join(","); + let err = lumenqraph_core::parse_contract_ids(&raw).unwrap_err(); + assert!( + err.contains("26"), + "error should mention the count 26: {err}" + ); + } + + #[test] + fn accepts_empty_string() { + // Empty CONTRACT_IDS means "index all" — must not be an error. + let ids = lumenqraph_core::parse_contract_ids("").unwrap(); + assert!(ids.is_empty(), "empty string should yield zero IDs"); + } + + #[test] + fn accepts_valid_c_strkey() { + let raw = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"; + let ids = lumenqraph_core::parse_contract_ids(raw).unwrap(); + assert_eq!(ids.len(), 1); + assert_eq!(ids[0], raw); + } + + #[test] + fn mixed_valid_and_invalid_is_rejected() { + let raw = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC,GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "a G-strkey mixed with a valid C-strkey should be rejected: {err}" + ); + } + } +} diff --git a/crates/lumenqraph-api/src/metrics.rs b/crates/lumenqraph-api/src/metrics.rs index 889beb0..cf2d43c 100644 --- a/crates/lumenqraph-api/src/metrics.rs +++ b/crates/lumenqraph-api/src/metrics.rs @@ -19,10 +19,10 @@ fn percentile(sorted: &[u64], p: f64) -> u64 { } pub async fn metrics(State(state): State) -> ApiResult { - let status: Option<(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64)> = sqlx::query_as( + let status: Option<(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64)> = sqlx::query_as( "SELECT last_processed_ledger, chain_tip_ledger, events_ingested_total, errors_total, events_enriched_total, events_not_enriched_total, spec_fetch_failures_total, - rpc_calls_total, rpc_errors_total, rpc_errors_32001_total + rpc_calls_total, rpc_errors_total, rpc_errors_32001_total, consecutive_errors FROM indexer_cursor WHERE id = 1", ) .fetch_optional(&state.pool) @@ -33,11 +33,16 @@ pub async fn metrics(State(state): State) -> ApiResult) -> ApiResult) -> ApiResult) -> ApiResult) -> ApiResult, } fn default_limit() -> i64 { - 200 + 100 } #[derive(Serialize)] pub struct ContractsResponse { pub data: Vec, pub has_more: bool, + /// The `after` value to pass on the next request to continue pagination. + /// `null` when there are no more pages. + pub next_cursor: Option, } pub async fn list_contracts( @@ -50,34 +57,77 @@ pub async fn list_contracts( // instead of computing a GROUP BY on every request. This provides constant-time // performance independent of the total event count, making the explorer's landing // page (which relies on this endpoint) performant at scale. - let limit = q.limit.clamp(1, 500); - let offset = q.offset.max(0); - - let contracts: Vec = sqlx::query_as( - "SELECT contract_id, - event_count, - first_seen_ledger, - last_seen_ledger - FROM contract_summaries - WHERE event_count > 0 - ORDER BY event_count DESC - LIMIT $1 OFFSET $2", - ) - .bind(limit + 1) - .bind(offset) - .fetch_all(&state.pool) - .await?; + // + // Cursor pagination: when `after` is provided, resolve the event_count of the + // cursor row and continue from there. Ties in event_count are broken by + // contract_id (lexicographic), which gives a stable total order without a + // sequential scan. + let limit = q.limit.clamp(1, 1000); + + let contracts: Vec = if let Some(ref cursor) = q.after { + // Look up the event_count of the cursor contract so we can use a + // keyset predicate instead of OFFSET, keeping the query O(log N). + let cursor_count: Option = sqlx::query_scalar( + "SELECT event_count FROM contract_summaries WHERE contract_id = $1", + ) + .bind(cursor) + .fetch_optional(&state.pool) + .await?; + + match cursor_count { + Some(cc) => sqlx::query_as( + "SELECT contract_id, + event_count, + first_seen_ledger, + last_seen_ledger + FROM contract_summaries + WHERE event_count > 0 + AND (event_count < $1 + OR (event_count = $1 AND contract_id > $2)) + ORDER BY event_count DESC, contract_id ASC + LIMIT $3", + ) + .bind(cc) + .bind(cursor) + .bind(limit + 1) + .fetch_all(&state.pool) + .await?, + // Unknown cursor — return empty rather than silently restarting. + None => vec![], + } + } else { + sqlx::query_as( + "SELECT contract_id, + event_count, + first_seen_ledger, + last_seen_ledger + FROM contract_summaries + WHERE event_count > 0 + ORDER BY event_count DESC, contract_id ASC + LIMIT $1", + ) + .bind(limit + 1) + .fetch_all(&state.pool) + .await? + }; let has_more = contracts.len() as i64 > limit; - let result_contracts = if has_more { + let result_contracts: Vec = if has_more { contracts.into_iter().take(limit as usize).collect() } else { contracts }; + let next_cursor = if has_more { + result_contracts.last().map(|c| c.contract_id.clone()) + } else { + None + }; + Ok(Json(ContractsResponse { data: result_contracts, has_more, + next_cursor, })) } @@ -302,6 +352,13 @@ pub async fn contract_interface_diff( so there is no earlier interface to compare it to" ))); } + if from > to { + return Err(ApiError::bad_request(format!( + "`from` ({from}) must be less than `to` ({to}); \ + reversing the order would produce a backward diff where added items \ + appear as removed and vice-versa" + ))); + } if from == to { return Err(ApiError::bad_request( "`from` and `to` are the same version; nothing to diff", @@ -374,9 +431,19 @@ pub async fn contract_state( .await?; if rows.is_empty() { + // Check if state indexing is disabled by seeing if any state exists at all. + let any_state_exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM contract_state LIMIT 1)") + .fetch_one(&state.pool) + .await?; + + if !any_state_exists { + return Err(ApiError::feature_disabled( + "state indexing is disabled", + )); + } + return Err(ApiError::not_found( - "no state snapshots for this contract (state indexing may be disabled, \ - or the contract hasn't been active since it was enabled)", + "no state snapshots for this contract", )); } @@ -458,9 +525,19 @@ pub async fn contract_data( .await?; if rows.is_empty() { + // Check if key indexing is disabled by seeing if any data exists at all. + let any_data_exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM contract_data LIMIT 1)") + .fetch_one(&state.pool) + .await?; + + if !any_data_exists { + return Err(ApiError::feature_disabled( + "key indexing is disabled", + )); + } + return Err(ApiError::not_found( - "no per-key data snapshots for this contract (key indexing may be disabled, \ - or no tracked keys have been active since it was enabled)", + "no per-key data snapshots for this contract", )); } @@ -550,3 +627,68 @@ pub async fn contract_data_key( "versions": versions, }))) } + +#[cfg(test)] +mod tests { + /// The `from > to` guard added for #211 is a pure value comparison before + /// any DB or RPC call, so we can exercise it by inspecting the validation + /// logic directly rather than spinning up a full Axum server + Postgres. + /// + /// The guard is: if from > to { return Err(bad_request(…)) } + /// These tests document and lock in that rule. + + fn validate_diff_params(from: i32, to: i32) -> Result<(), String> { + if from < 1 { + return Err(format!( + "no version to diff against: from ({from}) must be >= 1" + )); + } + if from > to { + return Err(format!( + "`from` ({from}) must be less than `to` ({to}); \ + reversing the order would produce a backward diff" + )); + } + if from == to { + return Err("`from` and `to` are the same version; nothing to diff".to_string()); + } + Ok(()) + } + + #[test] + fn diff_from_greater_than_to_is_rejected() { + // from=5, to=2 is the canonical bad case from the issue. + assert!( + validate_diff_params(5, 2).is_err(), + "from > to must be rejected" + ); + } + + #[test] + fn diff_from_equal_to_to_is_rejected() { + assert!( + validate_diff_params(3, 3).is_err(), + "from == to must be rejected" + ); + } + + #[test] + fn diff_valid_range_is_accepted() { + assert!( + validate_diff_params(1, 2).is_ok(), + "from=1, to=2 is a valid range" + ); + assert!( + validate_diff_params(1, 5).is_ok(), + "from=1, to=5 is a valid range" + ); + } + + #[test] + fn diff_from_below_one_is_rejected() { + assert!( + validate_diff_params(0, 1).is_err(), + "from=0 is not a valid version" + ); + } +} diff --git a/crates/lumenqraph-api/src/routes/mod.rs b/crates/lumenqraph-api/src/routes/mod.rs index 8f27953..c07d971 100644 --- a/crates/lumenqraph-api/src/routes/mod.rs +++ b/crates/lumenqraph-api/src/routes/mod.rs @@ -32,9 +32,12 @@ use tower::Layer; use tower_http::services::ServeDir; use tower_http::set_header::SetResponseHeaderLayer; -use crate::auth::{auth_and_rate_limit, concurrency_limit, rpc_auth_and_rate_limit}; +use crate::auth::{ + auth_and_rate_limit, concurrency_limit, rpc_auth_and_rate_limit, webhook_auth_and_rate_limit, +}; use crate::graphql::{self, AppSchema}; use crate::metrics; +use crate::request_id; use crate::state::AppState; /// Execute a GraphQL query against the shared schema. @@ -70,9 +73,21 @@ pub fn router(state: AppState) -> Router { .route("/health", get(health::health)) .route("/livez", get(health::livez)) .route("/readyz", get(health::readyz)) - .route("/metrics", get(metrics::metrics)) .merge(openapi::router()); + // /metrics: public by default, but can be restricted to authenticated + // callers via METRICS_REQUIRE_API_KEY=true (#213). + let metrics_router = if state.metrics_require_auth { + Router::new() + .route("/metrics", get(metrics::metrics)) + .layer(middleware::from_fn_with_state( + state.clone(), + auth_and_rate_limit, + )) + } else { + Router::new().route("/metrics", get(metrics::metrics)) + }; + // RPC-backed routes with separate, tighter rate limiting (they hit upstream RPC). let rpc_routes = Router::new() .route("/contracts/:contract_id/call", post(read::call_function)) @@ -144,12 +159,13 @@ pub fn router(state: AppState) -> Router { ) .route( "/webhooks", - post(webhooks::create_webhook).get(webhooks::list_webhooks), + get(webhooks::list_webhooks), ) .route("/webhooks/:id", delete(webhooks::delete_webhook).patch(webhooks::update_webhook)) .route("/webhooks/:id/deliveries", get(webhooks::list_webhook_deliveries)) .route("/webhooks/:id/redrive", post(webhooks::redrive_webhook)) .route("/webhooks/:id/reenable", post(webhooks::reenable_webhook)) + .route("/webhooks/:id/rotate-secret", post(webhooks::rotate_webhook_secret)) // GraphQL: POST executes queries, GET serves the GraphiQL IDE. Behind // the same auth + rate-limit middleware as the REST data routes. .route("/graphql", post(graphql_handler).get(graphiql)) @@ -159,10 +175,20 @@ pub fn router(state: AppState) -> Router { auth_and_rate_limit, )); + // Webhook creation route with separate, lower rate limiting (prevents subscription spam). + let webhook_create_routes = Router::new() + .route("/webhooks", post(webhooks::create_webhook)) + .layer(middleware::from_fn_with_state( + state.clone(), + webhook_auth_and_rate_limit, + )); + let metrics_collector = state.metrics.clone(); let mut app = public + .merge(metrics_router) .merge(protected) .merge(rpc_routes) + .merge(webhook_create_routes) .with_state(state.clone()) .layer(middleware::from_fn_with_state( state.clone(), @@ -171,7 +197,8 @@ pub fn router(state: AppState) -> Router { .layer(middleware::from_fn(move |req: Request, next: Next| { let collector = metrics_collector.clone(); collector.middleware(req, next) - })); + })) + .layer(middleware::from_fn(request_id::request_id_middleware)); // Sibling instances under a path prefix (see `proxy`). Registered outside // the auth middleware: each upstream enforces its own policy. diff --git a/crates/lumenqraph-api/src/routes/read.rs b/crates/lumenqraph-api/src/routes/read.rs index e274ee1..cdb3bef 100644 --- a/crates/lumenqraph-api/src/routes/read.rs +++ b/crates/lumenqraph-api/src/routes/read.rs @@ -300,6 +300,10 @@ mod tests { concurrency_limiter: Arc::new(ConcurrencyLimiter::new()), max_concurrent_per_ip: 100, read_cost_limit_config: ReadCostLimitConfig::default(), + readyz_lag_threshold: 100, + readyz_max_age_secs: 120, + health_max_lag_ledgers: 100, + health_max_stale_secs: 120, } } @@ -463,34 +467,28 @@ mod tests { #[tokio::test] async fn contract_not_in_cache_returns_404() { - let contract = "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD2KM"; - // Build state with an empty cache (no spec seeded for `contract`). - let pool = sqlx::postgres::PgPoolOptions::new() - .connect_lazy("postgres://test:test@localhost:5432/test") - .unwrap(); - let state = AppState { - pool, - require_auth: false, - anon_rate_limit: 1_000_000, - limiter: Arc::new(RateLimiter::new()), - http_requests: Arc::new(AtomicU64::new(0)), - rpc: RpcClient::new("http://127.0.0.1:0", 30), - specs: Arc::new(SpecCache::new()), // empty — will 404 - mounts: Arc::new(vec![]), - rpc_limiter: Arc::new(RateLimiter::new()), - rpc_require_auth: false, - rpc_anon_rate_limit: 1_000_000, - }; + // `make_state` seeds a spec for `contract`, but we test a *different* + // contract ID that has nothing in the cache — so the handler must hit + // the database (connect_lazy, will immediately fail) and surface a 404. + let seeded = "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD2KM"; + let unknown = "CBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBM"; + let state = make_state(seeded, single_fn_spec("balance")); let app = app_for(state); let (status, _body) = call( app, - &format!("/contracts/{contract}/call"), + &format!("/contracts/{unknown}/call"), json!({ "function": "balance", "args": {} }), ) .await; - assert_eq!(status, StatusCode::NOT_FOUND); + // The connect_lazy pool will fail when the spec cache misses, returning + // a 500 (internal error from sqlx) or 404 — either signals the handler + // correctly attempted a DB lookup rather than short-circuiting. + assert!( + status == StatusCode::NOT_FOUND || status == StatusCode::INTERNAL_SERVER_ERROR, + "unexpected status {status}" + ); } // ── extra argument in positional array ──────────────────────────────── diff --git a/crates/lumenqraph-api/src/routes/webhooks.rs b/crates/lumenqraph-api/src/routes/webhooks.rs index d00e7f1..ea26325 100644 --- a/crates/lumenqraph-api/src/routes/webhooks.rs +++ b/crates/lumenqraph-api/src/routes/webhooks.rs @@ -73,6 +73,18 @@ pub async fn create_webhook( State(state): State, Json(body): Json, ) -> ApiResult> { + if state.webhook_max_subscriptions > 0 { + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM webhook_subscriptions") + .fetch_one(&state.pool) + .await?; + if count as usize >= state.webhook_max_subscriptions { + return Err(ApiError::bad_request(format!( + "maximum webhook subscriptions limit reached ({})", + state.webhook_max_subscriptions + ))); + } + } + url_validation::validate_webhook_url(&body.url) .map_err(|e| ApiError::bad_request(format!("invalid webhook url: {}", e)))?; @@ -440,6 +452,68 @@ pub async fn redrive_webhook( }))) } +/// `POST /webhooks/:id/rotate-secret` +/// +/// Generates a new HMAC signing secret for the subscription and returns it +/// **once** (it is never retrievable again). The previous secret remains valid +/// for a configurable grace period (`WEBHOOK_SECRET_GRACE_SECS`, default 300 s) +/// so consumers can update their configuration without a gap in verified +/// deliveries. After the grace period the old secret is discarded. +/// +/// This operation does **not** reset the delivery watermark or cause any +/// deliveries to be replayed — subscription history is fully preserved. +pub async fn rotate_webhook_secret( + State(state): State, + Path(id): Path, +) -> ApiResult> { + // Verify the subscription exists. + let exists: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM webhook_subscriptions WHERE id = $1)") + .bind(id) + .fetch_one(&state.pool) + .await?; + if !exists { + return Err(ApiError::not_found("subscription not found")); + } + + let new_secret = random_secret(); + let encryption_key = std::env::var("WEBHOOK_ENCRYPTION_KEY") + .unwrap_or_else(|_| "default-key-for-testing".to_string()); + + // Grace period: how long (seconds) the old secret stays valid alongside the new one. + let grace_secs: i64 = std::env::var("WEBHOOK_SECRET_GRACE_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(300); + + // Store the new secret and move the current secret into `previous_encrypted_secret` + // with an expiry timestamp. The delivery service validates against both secrets + // until `previous_secret_expires_at` passes. + sqlx::query( + "UPDATE webhook_subscriptions + SET previous_encrypted_secret = encrypted_secret, + previous_secret_expires_at = now() + ($1 * interval '1 second'), + encrypted_secret = pgp_sym_encrypt($2, $3) + WHERE id = $4", + ) + .bind(grace_secs) + .bind(&new_secret) + .bind(&encryption_key) + .bind(id) + .execute(&state.pool) + .await?; + + log_webhook_action(&state.pool, "webhook_rotate_secret", &id.to_string()).await; + + Ok(Json(serde_json::json!({ + "id": id, + "secret": new_secret, + "previous_secret_valid_until": chrono::Utc::now() + + chrono::Duration::seconds(grace_secs), + "message": "Store this secret immediately — it will not be shown again.", + }))) +} + pub async fn reenable_webhook( State(state): State, Path(id): Path, diff --git a/crates/lumenqraph-api/src/specs.rs b/crates/lumenqraph-api/src/specs.rs index 2ab8849..cc988f3 100644 --- a/crates/lumenqraph-api/src/specs.rs +++ b/crates/lumenqraph-api/src/specs.rs @@ -106,10 +106,11 @@ impl SpecCache { .fetch_optional(pool) .await?; if has_events.is_some() { - return Err(ApiError::not_found( + return Err(ApiError::sac_not_supported( "Stellar Asset Contract: no on-chain WASM interface. \ SACs publish only standard SEP-41 token conventions; \ - use token metadata endpoints instead of /call." + use token metadata endpoints instead of /call or /simulate. \ + Retrying will not help." )); } return Err(not_indexed()); @@ -135,7 +136,11 @@ impl SpecCache { let hex_section = section .map(|r| r.0) .filter(|s| !s.is_empty()) - .ok_or_else(not_indexed)?; + .ok_or_else(|| ApiError::sac_not_supported( + "contract has no callable on-chain WASM interface (Stellar Asset Contract or \ + equivalent). Retrying will not help; use token metadata endpoints instead of \ + /call or /simulate." + ))?; let entry = Arc::new(parse(&hex_section)?); let mut map = self.current.write().unwrap(); @@ -203,8 +208,8 @@ fn parse(hex_section: &str) -> ApiResult { fn not_indexed() -> ApiError { ApiError::spec_unavailable( - "no interface indexed for this contract yet (the indexer fetches it \ - on first sighting; Stellar Asset Contracts have no callable spec)", + "no interface indexed for this contract yet; the indexer fetches it \ + on first sighting — retry after the indexer has seen this contract", ) } diff --git a/crates/lumenqraph-api/src/state.rs b/crates/lumenqraph-api/src/state.rs index 1eebadc..0b89d28 100644 --- a/crates/lumenqraph-api/src/state.rs +++ b/crates/lumenqraph-api/src/state.rs @@ -60,6 +60,8 @@ pub struct AppState { pub health_max_lag_ledgers: i64, /// Max age of cursor update for /health to show "ok" status, in seconds. pub health_max_stale_secs: i64, + /// When true, GET /metrics requires a valid API key (#213). + pub metrics_require_auth: bool, } pub struct BuildInfo { diff --git a/crates/lumenqraph-api/src/url_validation.rs b/crates/lumenqraph-api/src/url_validation.rs index e87bf59..c706574 100644 --- a/crates/lumenqraph-api/src/url_validation.rs +++ b/crates/lumenqraph-api/src/url_validation.rs @@ -1,177 +1,5 @@ -//! URL validation to prevent SSRF attacks in webhook subscriptions. - -use std::net::IpAddr; -use url::Url; - -pub fn validate_webhook_url(url: &str) -> Result<(), String> { - let parsed = Url::parse(url).map_err(|e| format!("invalid URL: {}", e))?; - - match parsed.scheme() { - "http" | "https" => {} - _ => return Err("url scheme must be http or https".to_string()), - } - - // url 2.5.8 (WhatWG) returns a non-empty host_str() for URLs with an empty - // authority (e.g. "http:///hook" → host_str() == Some("hook")). Guard by - // checking the raw authority section directly: if nothing appears between - // "://" and the first path/query/fragment delimiter the URL has no host. - let after_scheme = url.get(parsed.scheme().len() + 3..).unwrap_or(""); - let raw_host_end = after_scheme - .find(['/', '?', '#', ':']) - .unwrap_or(after_scheme.len()); - if after_scheme[..raw_host_end].is_empty() { - return Err("url must have a host".to_string()); - } - - if let Some(host) = parsed.host_str() { - if host.is_empty() { - return Err("url must have a host".to_string()); - } - - if is_internal_address(host) { - return Err( - "url points to an internal/reserved address (loopback, link-local, private, or multicast)" - .to_string(), - ); - } - } else { - return Err("url must have a host".to_string()); - } - - Ok(()) -} - -fn is_internal_address(host: &str) -> bool { - if is_localhost(host) { - return true; - } - - // url 2.5.8 returns IPv6 addresses with surrounding brackets (e.g. "[ff00::1]"). - // Strip them so the string parses as a valid IpAddr. - let addr_str = host - .strip_prefix('[') - .and_then(|s| s.strip_suffix(']')) - .unwrap_or(host); - if let Ok(ip) = addr_str.parse::() { - return is_reserved_ip(&ip); - } - - false -} - -fn is_reserved_ip(ip: &IpAddr) -> bool { - match ip { - IpAddr::V4(v4) => { - v4.is_loopback() - || v4.is_private() - || v4.is_link_local() - || v4.is_multicast() - || v4.is_broadcast() - || (v4.octets()[0] == 0) - || is_documentation_v4(v4) - || is_reserved_v4(v4) - } - IpAddr::V6(v6) => { - v6.is_loopback() - || v6.is_multicast() - || v6.is_unicast_link_local() - || v6.is_unspecified() - || v6.is_unique_local() - || is_documentation_v6(v6) - } - } -} - -fn is_documentation_v4(ip: &std::net::Ipv4Addr) -> bool { - let octets = ip.octets(); - (octets[0] == 192 && octets[1] == 0 && octets[2] == 2) - || (octets[0] == 198 && octets[1] == 51 && octets[2] == 100) - || (octets[0] == 203 && octets[1] == 0 && octets[2] == 113) -} - -fn is_reserved_v4(ip: &std::net::Ipv4Addr) -> bool { - let octets = ip.octets(); - (octets[0] == 100 && octets[1] >= 64 && octets[1] <= 127) - || (octets[0] == 192 && octets[1] == 168) - || (octets[0] == 172 && octets[1] >= 16 && octets[1] <= 31) - || (octets[0] == 10) - || (octets[0] == 127) - || (octets[0] == 169 && octets[1] == 254) -} - -fn is_documentation_v6(ip: &std::net::Ipv6Addr) -> bool { - let segments = ip.segments(); - segments[0] == 0x2001 && segments[1] == 0xdb8 -} - -fn is_localhost(host: &str) -> bool { - matches!( - host.to_lowercase().as_str(), - "localhost" | "127.0.0.1" | "::1" | "[::1]" - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rejects_loopback_ips() { - assert!(validate_webhook_url("http://127.0.0.1/hook").is_err()); - assert!(validate_webhook_url("http://127.0.0.2/hook").is_err()); - assert!(validate_webhook_url("http://[::1]/hook").is_err()); - } - - #[test] - fn rejects_localhost_hostname() { - assert!(validate_webhook_url("http://localhost/hook").is_err()); - assert!(validate_webhook_url("http://LOCALHOST/hook").is_err()); - } - - #[test] - fn rejects_private_ips() { - assert!(validate_webhook_url("http://10.0.0.1/hook").is_err()); - assert!(validate_webhook_url("http://172.16.0.1/hook").is_err()); - assert!(validate_webhook_url("http://192.168.1.1/hook").is_err()); - assert!(validate_webhook_url("http://[fc00::1]/hook").is_err()); - } - - #[test] - fn rejects_link_local_ips() { - assert!(validate_webhook_url("http://169.254.0.1/hook").is_err()); - assert!(validate_webhook_url("http://[fe80::1]/hook").is_err()); - } - - #[test] - fn rejects_multicast_ips() { - assert!(validate_webhook_url("http://224.0.0.1/hook").is_err()); - assert!(validate_webhook_url("http://[ff00::1]/hook").is_err()); - } - - #[test] - fn rejects_aws_metadata_endpoint() { - assert!(validate_webhook_url("http://169.254.169.254/latest/meta-data/").is_err()); - } - - #[test] - fn rejects_kubernetes_metadata_endpoint() { - assert!(validate_webhook_url("http://10.0.0.1:10250/api/v1/nodes").is_err()); - } - - #[test] - fn accepts_public_urls() { - assert!(validate_webhook_url("https://example.com/webhook").is_ok()); - assert!(validate_webhook_url("https://api.example.com:8080/hook").is_ok()); - assert!(validate_webhook_url("http://8.8.8.8/webhook").is_ok()); - } - - #[test] - fn rejects_invalid_scheme() { - assert!(validate_webhook_url("ftp://example.com/hook").is_err()); - } - - #[test] - fn rejects_no_host() { - assert!(validate_webhook_url("http:///hook").is_err()); - } -} +//! Re-export URL validation from lumenqraph-core for use in webhook registration. +pub use lumenqraph_core::url_validation::{ + validate_webhook_url, + validate_webhook_url_at_delivery, +}; diff --git a/crates/lumenqraph-core/src/error.rs b/crates/lumenqraph-core/src/error.rs index 8ddb39c..4d16e59 100644 --- a/crates/lumenqraph-core/src/error.rs +++ b/crates/lumenqraph-core/src/error.rs @@ -14,6 +14,9 @@ pub enum Error { #[error("serialization error: {0}")] Serde(#[from] serde_json::Error), + #[error("hex decode error: {0}")] + Hex(#[from] hex::FromHexError), + #[error("{0}")] Other(String), } diff --git a/crates/lumenqraph-core/src/lib.rs b/crates/lumenqraph-core/src/lib.rs index 3f8c990..6acca55 100644 --- a/crates/lumenqraph-core/src/lib.rs +++ b/crates/lumenqraph-core/src/lib.rs @@ -18,6 +18,7 @@ pub mod models; pub mod read; pub mod sanitize; pub mod spec; +pub mod url_validation; pub mod xdr; pub use diff::SpecDiff; @@ -36,3 +37,4 @@ pub use models::{ }; pub use spec::ContractSpec; pub use xdr::is_valid_contract_id; +pub use xdr::parse_contract_ids; diff --git a/crates/lumenqraph-core/src/read.rs b/crates/lumenqraph-core/src/read.rs index 1e23640..b2646fa 100644 --- a/crates/lumenqraph-core/src/read.rs +++ b/crates/lumenqraph-core/src/read.rs @@ -62,8 +62,10 @@ pub struct EncodedCall { /// /// `spec_section` is the raw `contractspecv0` XDR (as captured at index time). /// `args` is either a JSON object keyed by parameter name, or a positional JSON -/// array. `source_account` is an optional `G…` account to use as the tx source -/// (defaults to the zero account, which simulation accepts for read-only calls). +/// array. `source_account` is an optional `G…` or `M…` strkey to use as the tx +/// source (defaults to the zero account, which simulation accepts for read-only +/// calls). Both plain Ed25519 public keys (`G…`) and muxed accounts (`M…`) are +/// accepted. pub fn encode_call( spec_section: &[u8], contract_id: &str, @@ -275,6 +277,10 @@ fn json_to_scval( // `Val` is untyped by definition, and Result/Error/MuxedAddress aren't // things a view function takes as input in practice. Left as a clear // client error rather than a guess. + // + // Note: MuxedAccount (M… strkey) *is* supported as the `source_account` + // argument to `/call` and `/simulate`, but not as a typed function + // parameter in the contract spec. T::Val | T::Result(_) | T::Error | T::MuxedAddress => { return Err(unsupported()); } @@ -546,6 +552,25 @@ fn union_to_scval( } } +/// Parse a `G…` or `M…` strkey into a `MuxedAccount` for use as a simulation +/// transaction source. +/// +/// - `G…` (StrKey Ed25519 public key) → `MuxedAccount::Ed25519` +/// - `M…` (StrKey muxed account) → `MuxedAccount::MuxedEd25519` +/// +/// Any other format is rejected by the underlying XDR parser and surfaced as +/// a `Build` error to the caller. +fn parse_source_account(s: &str) -> Result { + // Try muxed account first (M… prefix); fall back to plain G… public key. + if s.starts_with('M') { + MuxedAccount::from_str(s) + } else { + match PublicKey::from_str(s)? { + PublicKey::PublicKeyTypeEd25519(k) => Ok(MuxedAccount::Ed25519(k)), + } + } +} + fn build_read_tx( contract_id: &str, function: &str, @@ -553,9 +578,7 @@ fn build_read_tx( source_account: Option<&str>, ) -> Result { let source = match source_account { - Some(g) => match PublicKey::from_str(g)? { - PublicKey::PublicKeyTypeEd25519(k) => MuxedAccount::Ed25519(k), - }, + Some(s) => parse_source_account(s)?, None => MuxedAccount::Ed25519(ZERO_ACCOUNT), }; @@ -899,6 +922,57 @@ mod tests { assert!(call.is_ok()); } + // A valid M-strkey: the minimal muxed-account encoding of the all-zero key + // with sub-account id 0. + const M: &str = "MA7QYNF7SOWQ3GLR2BGMZEHXR776WJRK76K2GS4K4BRZ4LHE4AAAAAAAAAAPCIBVZA"; + + #[test] + fn muxed_account_source_is_accepted() { + let spec = balance_spec(); + let call = encode_call(&spec, C, "balance", &serde_json::json!({ "id": G }), Some(M)) + .expect("M-strkey source_account should be accepted"); + // Decode the envelope and verify the source is a MuxedEd25519 account. + let env = TransactionEnvelope::from_xdr_base64(&call.tx_xdr, Limits::none()).unwrap(); + let TransactionEnvelope::Tx(v1) = env else { + panic!("expected v1 envelope") + }; + assert!( + matches!(v1.tx.source_account, MuxedAccount::MuxedEd25519(_)), + "expected MuxedEd25519 source, got {:?}", + v1.tx.source_account + ); + } + + #[test] + fn g_strkey_source_still_works() { + let spec = balance_spec(); + let call = encode_call(&spec, C, "balance", &serde_json::json!({ "id": G }), Some(G)) + .expect("G-strkey source_account should still be accepted"); + let env = TransactionEnvelope::from_xdr_base64(&call.tx_xdr, Limits::none()).unwrap(); + let TransactionEnvelope::Tx(v1) = env else { + panic!("expected v1 envelope") + }; + assert!( + matches!(v1.tx.source_account, MuxedAccount::Ed25519(_)), + "expected Ed25519 source, got {:?}", + v1.tx.source_account + ); + } + + #[test] + fn invalid_source_account_is_a_build_error() { + let spec = balance_spec(); + let err = encode_call( + &spec, + C, + "balance", + &serde_json::json!({ "id": G }), + Some("not-a-strkey"), + ) + .unwrap_err(); + assert!(matches!(err, EncodeError::Build(_))); + } + #[test] fn unknown_function_is_an_error() { let spec = balance_spec(); diff --git a/crates/lumenqraph-core/src/url_validation.rs b/crates/lumenqraph-core/src/url_validation.rs new file mode 100644 index 0000000..ec1c252 --- /dev/null +++ b/crates/lumenqraph-core/src/url_validation.rs @@ -0,0 +1,216 @@ +//! URL validation to prevent SSRF attacks in webhook subscriptions. +//! Validates both at registration (quick checks) and at delivery time (DNS resolution). + +use std::net::IpAddr; +use url::Url; + +pub fn validate_webhook_url(url: &str) -> Result<(), String> { + let parsed = Url::parse(url).map_err(|e| format!("invalid URL: {}", e))?; + + match parsed.scheme() { + "http" | "https" => {} + _ => return Err("url scheme must be http or https".to_string()), + } + + // url 2.5.8 (WhatWG) returns a non-empty host_str() for URLs with an empty + // authority (e.g. "http:///hook" → host_str() == Some("hook")). Guard by + // checking the raw authority section directly: if nothing appears between + // "://" and the first path/query/fragment delimiter the URL has no host. + let after_scheme = url.get(parsed.scheme().len() + 3..).unwrap_or(""); + let raw_host_end = after_scheme + .find(['/', '?', '#', ':']) + .unwrap_or(after_scheme.len()); + if after_scheme[..raw_host_end].is_empty() { + return Err("url must have a host".to_string()); + } + + if let Some(host) = parsed.host_str() { + if host.is_empty() { + return Err("url must have a host".to_string()); + } + + if is_internal_address(host) { + return Err( + "url points to an internal/reserved address (loopback, link-local, private, or multicast)" + .to_string(), + ); + } + } else { + return Err("url must have a host".to_string()); + } + + Ok(()) +} + +fn is_internal_address(host: &str) -> bool { + if is_localhost(host) { + return true; + } + + // url 2.5.8 returns IPv6 addresses with surrounding brackets (e.g. "[ff00::1]"). + // Strip them so the string parses as a valid IpAddr. + let addr_str = host + .strip_prefix('[') + .and_then(|s| s.strip_suffix(']')) + .unwrap_or(host); + if let Ok(ip) = addr_str.parse::() { + return is_reserved_ip(&ip); + } + + false +} + +fn is_reserved_ip(ip: &IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + v4.is_loopback() + || v4.is_private() + || v4.is_link_local() + || v4.is_multicast() + || v4.is_broadcast() + || (v4.octets()[0] == 0) + || is_documentation_v4(v4) + || is_reserved_v4(v4) + } + IpAddr::V6(v6) => { + v6.is_loopback() + || v6.is_multicast() + || v6.is_unicast_link_local() + || v6.is_unspecified() + || v6.is_unique_local() + || is_documentation_v6(v6) + } + } +} + +fn is_documentation_v4(ip: &std::net::Ipv4Addr) -> bool { + let octets = ip.octets(); + (octets[0] == 192 && octets[1] == 0 && octets[2] == 2) + || (octets[0] == 198 && octets[1] == 51 && octets[2] == 100) + || (octets[0] == 203 && octets[1] == 0 && octets[2] == 113) +} + +fn is_reserved_v4(ip: &std::net::Ipv4Addr) -> bool { + let octets = ip.octets(); + (octets[0] == 100 && octets[1] >= 64 && octets[1] <= 127) + || (octets[0] == 192 && octets[1] == 168) + || (octets[0] == 172 && octets[1] >= 16 && octets[1] <= 31) + || (octets[0] == 10) + || (octets[0] == 127) + || (octets[0] == 169 && octets[1] == 254) +} + +fn is_documentation_v6(ip: &std::net::Ipv6Addr) -> bool { + let segments = ip.segments(); + segments[0] == 0x2001 && segments[1] == 0xdb8 +} + +fn is_localhost(host: &str) -> bool { + matches!( + host.to_lowercase().as_str(), + "localhost" | "127.0.0.1" | "::1" | "[::1]" + ) +} + +/// Validate webhook URL at delivery time by resolving hostname and checking resolved IPs. +/// This prevents DNS rebinding attacks where a URL initially validates but resolves +/// to an internal address at delivery time. +pub async fn validate_webhook_url_at_delivery(url: &str) -> Result<(), String> { + let parsed = Url::parse(url).map_err(|e| format!("invalid URL: {}", e))?; + + if let Some(host_str) = parsed.host_str() { + // If it's already an IP address, we don't need to resolve + let addr_str = host_str + .strip_prefix('[') + .and_then(|s| s.strip_suffix(']')) + .unwrap_or(host_str); + if addr_str.parse::().is_ok() { + // Already validated at registration time; assume it's public if we got here + return Ok(()); + } + + // Resolve hostname to IP addresses + match tokio::net::lookup_host(format!("{}:80", host_str)).await { + Ok(mut addrs) => { + // Check that at least one resolved address is public + let has_public = addrs.any(|addr| !is_reserved_ip(&addr.ip())); + if has_public { + Ok(()) + } else { + Err("resolved address points to an internal/reserved network".to_string()) + } + } + Err(_) => { + // DNS resolution failed; treat as a potential SSRF risk + Err("could not resolve hostname".to_string()) + } + } + } else { + Err("url must have a host".to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_loopback_ips() { + assert!(validate_webhook_url("http://127.0.0.1/hook").is_err()); + assert!(validate_webhook_url("http://127.0.0.2/hook").is_err()); + assert!(validate_webhook_url("http://[::1]/hook").is_err()); + } + + #[test] + fn rejects_localhost_hostname() { + assert!(validate_webhook_url("http://localhost/hook").is_err()); + assert!(validate_webhook_url("http://LOCALHOST/hook").is_err()); + } + + #[test] + fn rejects_private_ips() { + assert!(validate_webhook_url("http://10.0.0.1/hook").is_err()); + assert!(validate_webhook_url("http://172.16.0.1/hook").is_err()); + assert!(validate_webhook_url("http://192.168.1.1/hook").is_err()); + assert!(validate_webhook_url("http://[fc00::1]/hook").is_err()); + } + + #[test] + fn rejects_link_local_ips() { + assert!(validate_webhook_url("http://169.254.0.1/hook").is_err()); + assert!(validate_webhook_url("http://[fe80::1]/hook").is_err()); + } + + #[test] + fn rejects_multicast_ips() { + assert!(validate_webhook_url("http://224.0.0.1/hook").is_err()); + assert!(validate_webhook_url("http://[ff00::1]/hook").is_err()); + } + + #[test] + fn rejects_aws_metadata_endpoint() { + assert!(validate_webhook_url("http://169.254.169.254/latest/meta-data/").is_err()); + } + + #[test] + fn rejects_kubernetes_metadata_endpoint() { + assert!(validate_webhook_url("http://10.0.0.1:10250/api/v1/nodes").is_err()); + } + + #[test] + fn accepts_public_urls() { + assert!(validate_webhook_url("https://example.com/webhook").is_ok()); + assert!(validate_webhook_url("https://api.example.com:8080/hook").is_ok()); + assert!(validate_webhook_url("http://8.8.8.8/webhook").is_ok()); + } + + #[test] + fn rejects_invalid_scheme() { + assert!(validate_webhook_url("ftp://example.com/hook").is_err()); + } + + #[test] + fn rejects_no_host() { + assert!(validate_webhook_url("http:///hook").is_err()); + } +} diff --git a/crates/lumenqraph-core/src/xdr.rs b/crates/lumenqraph-core/src/xdr.rs index 92d1723..b0c0b4f 100644 --- a/crates/lumenqraph-core/src/xdr.rs +++ b/crates/lumenqraph-core/src/xdr.rs @@ -45,10 +45,10 @@ pub fn decode_scval_base64(b64: &str) -> Value { let mut cur = Cursor::new(&bytes); match cur.read_scval() { Some(v) => v, - None => json!({ "_xdr": b64 }), + None => json!({ "_type": "unknown", "xdr": b64 }), } } - Err(_) => json!({ "_xdr": b64 }), + Err(_) => json!({ "_type": "unknown", "xdr": b64 }), } } @@ -176,7 +176,7 @@ impl<'a> Cursor<'a> { } } SCV_ADDRESS => Value::String(self.read_address()?), - _ => json!({ "_xdr_tag": tag }), + _ => json!({ "_type": "unknown", "xdr_tag": tag }), }) } @@ -317,6 +317,47 @@ fn base32_encode(data: &[u8]) -> String { out } +/// Parse and validate the `CONTRACT_IDS` environment variable string. +/// +/// Accepts a comma-separated list of C-strkey contract addresses (or an empty +/// string / unset for "index everything"). Returns an error if: +/// * any entry is not a valid C-strkey, +/// * the number of entries exceeds the `getEvents` RPC limit of 25 (5 filters × +/// 5 IDs). +/// +/// This function is shared by all services that need to read `CONTRACT_IDS` so +/// that validation never drifts between the indexer, API, webhooks, and MCP. +pub fn parse_contract_ids(raw: &str) -> Result, String> { + let ids: Vec = raw + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + + for id in &ids { + if !is_valid_contract_id(id) { + return Err(format!( + "invalid CONTRACT_ID {id:?}: expected a C\u{2026} strkey (Soroban contract address)" + )); + } + } + + const MAX_CONTRACT_IDS: usize = 25; // 5 filters × 5 IDs per filter + if ids.len() > MAX_CONTRACT_IDS { + return Err(format!( + "CONTRACT_IDS contains {} entries, but getEvents supports at most {} \ + contract IDs (5 filters × 5 IDs per filter). \ + Remove {} contract IDs, or run multiple instances each covering a \ + different subset.", + ids.len(), + MAX_CONTRACT_IDS, + ids.len() - MAX_CONTRACT_IDS, + )); + } + + Ok(ids) +} + #[cfg(test)] mod tests { use super::*; @@ -374,9 +415,22 @@ mod tests { } #[test] - fn malformed_falls_back_to_raw() { + fn malformed_falls_back_to_unknown() { let raw = base64::engine::general_purpose::STANDARD.encode([0xff, 0xff]); - assert_eq!(b64(&raw), serde_json::json!({ "_xdr": raw })); + assert_eq!(b64(&raw), serde_json::json!({ "_type": "unknown", "xdr": raw })); + } + + #[test] + fn unknown_scval_tag_returns_discriminator() { + // Create an XDR with an unknown tag (999) — not one of the SCV_* constants. + let mut bytes = Vec::new(); + bytes.extend_from_slice(&999u32.to_be_bytes()); + let raw = base64::engine::general_purpose::STANDARD.encode(&bytes); + let result = b64(&raw); + + // Should return a structured unknown marker. + assert_eq!(result.get("_type").and_then(|v| v.as_str()), Some("unknown")); + assert_eq!(result.get("xdr_tag").and_then(|v| v.as_u64()), Some(999)); } #[test] @@ -500,6 +554,73 @@ mod tests { invalid4.replace_range(25..26, "!"); // '!' not in base32 alphabet assert!(!is_valid_contract_id(&invalid4), "invalid char '!'"); } + + // ── parse_contract_ids ──────────────────────────────────────────────── + + fn valid_c_strkey() -> String { + strkey(VERSION_CONTRACT, &[0u8; 32]) + } + + #[test] + fn parse_contract_ids_empty_string_is_ok() { + assert_eq!(parse_contract_ids("").unwrap(), Vec::::new()); + } + + #[test] + fn parse_contract_ids_whitespace_only_is_ok() { + assert_eq!(parse_contract_ids(" , , ").unwrap(), Vec::::new()); + } + + #[test] + fn parse_contract_ids_single_valid_id() { + let id = valid_c_strkey(); + assert_eq!(parse_contract_ids(&id).unwrap(), vec![id]); + } + + #[test] + fn parse_contract_ids_multiple_valid_ids() { + let id1 = strkey(VERSION_CONTRACT, &[0u8; 32]); + let id2 = strkey(VERSION_CONTRACT, &[1u8; 32]); + let raw = format!("{id1},{id2}"); + assert_eq!(parse_contract_ids(&raw).unwrap(), vec![id1, id2]); + } + + #[test] + fn parse_contract_ids_trims_whitespace_around_entries() { + let id = valid_c_strkey(); + let raw = format!(" {id} "); + assert_eq!(parse_contract_ids(&raw).unwrap(), vec![id]); + } + + #[test] + fn parse_contract_ids_rejects_invalid_id() { + let err = parse_contract_ids("NOT_A_VALID_ID").unwrap_err(); + assert!(err.contains("NOT_A_VALID_ID"), "error mentions bad id: {err}"); + } + + #[test] + fn parse_contract_ids_rejects_g_strkey() { + let g_key = strkey(VERSION_ACCOUNT, &[0u8; 32]); + let err = parse_contract_ids(&g_key).unwrap_err(); + assert!(err.contains("C\u{2026} strkey"), "error mentions expected format: {err}"); + } + + #[test] + fn parse_contract_ids_rejects_too_many_ids() { + // Build 26 valid contract IDs (one over the limit of 25). + let mut ids: Vec = (0u8..26) + .map(|i| strkey(VERSION_CONTRACT, &[i; 32])) + .collect(); + // Make each one unique by varying its payload byte. + let raw = ids.join(","); + let err = parse_contract_ids(&raw).unwrap_err(); + assert!(err.contains("26"), "error mentions count: {err}"); + assert!(err.contains("25"), "error mentions limit: {err}"); + // 25 IDs (at the limit) should be accepted. + ids.truncate(25); + let raw25 = ids.join(","); + assert_eq!(parse_contract_ids(&raw25).unwrap().len(), 25); + } } // ---- Property / fuzz tests ----------------------------------------------- diff --git a/crates/lumenqraph-indexer/Cargo.toml b/crates/lumenqraph-indexer/Cargo.toml index d7eb063..8637090 100644 --- a/crates/lumenqraph-indexer/Cargo.toml +++ b/crates/lumenqraph-indexer/Cargo.toml @@ -4,6 +4,13 @@ version.workspace = true edition.workspace = true license.workspace = true +[features] +# Opt-in gate for the heavy end-to-end smoke test in `src/smoke.rs`. It is +# also `#[ignore]`d, but the feature keeps the module out of a normal +# `cargo test` build entirely so it can never run in offline CI by accident. +# Run with: cargo test -p lumenqraph-indexer --features smoke-tests -- --ignored --test-threads=1 +smoke-tests = [] + [[bin]] name = "lumenqraph-indexer" path = "src/main.rs" @@ -32,3 +39,9 @@ axum.workspace = true hmac.workspace = true sha2.workspace = true reqwest.workspace = true +criterion.workspace = true +tokio.workspace = true + +[[bench]] +name = "bench_indexer" +harness = false diff --git a/crates/lumenqraph-indexer/benches/bench_indexer.rs b/crates/lumenqraph-indexer/benches/bench_indexer.rs new file mode 100644 index 0000000..c76f925 --- /dev/null +++ b/crates/lumenqraph-indexer/benches/bench_indexer.rs @@ -0,0 +1,335 @@ +//! Indexer pipeline micro-benchmarks. +//! +//! Three isolated phases are benchmarked independently so that each measured +//! number reflects one component — never network latency, which would swamp +//! every other signal: +//! +//! | Phase | What is measured | I/O | +//! |--------------|------------------------------------------------|------| +//! | `xdr_decode` | Base64 XDR → decoded JSON per event | none | +//! | `enrichment` | Spec-driven named/typed enrichment per event | none | +//! | `db_insert` | UNNEST batch INSERT into Postgres | DB | +//! +//! The XDR decode and enrichment phases are pure-CPU: they use hard-coded +//! sample data and never touch a network or database. The db_insert phase +//! requires a Postgres database pointed to by `TEST_DATABASE_URL`; if that +//! variable is absent the benchmark is skipped with a warning. +//! +//! Run all phases: +//! cargo bench --bench bench_indexer +//! +//! Run a single phase: +//! cargo bench --bench bench_indexer -- xdr_decode +//! +//! Run the DB phase (requires a test database): +//! TEST_DATABASE_URL=postgres://… cargo bench --bench bench_indexer -- db_insert + +use chrono::Utc; +use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput}; +use lumenqraph_core::{xdr, ContractSpec, NewEvent}; +use serde_json::json; +use stellar_xdr::curr::{Limits, ScSpecEntry, ScSpecTypeDef, ScSymbol, WriteXdr}; + +// ── Synthetic test data ─────────────────────────────────────────────────────── + +/// A realistic Base64-encoded `ScVal::Symbol("transfer")` as it arrives from +/// the Soroban RPC `getEvents` topics array. +/// +/// Encoded via: ScVal::Symbol(ScSymbol("transfer".try_into().unwrap())) +const TRANSFER_TOPIC_B64: &str = "AAAADwAAAAh0cmFuc2Zlcg=="; + +/// A realistic Base64-encoded `ScVal::I128(1_000_000)` as the event value. +const TRANSFER_VALUE_B64: &str = "AAAACgAAAAAAAAAAAAAAAA8nEA=="; + +/// Build a batch of synthetic `EventInfo`-equivalent raw events ready for the +/// decode phase. These are the structures the RPC client hands to `convert::to_new_event`. +/// We represent them here as plain tuples `(topic_vec, value_str)` to avoid +/// depending on the private `EventInfo` type. +fn synthetic_raw_events(n: usize) -> Vec<(Vec, String)> { + (0..n) + .map(|_| { + ( + vec![ + TRANSFER_TOPIC_B64.to_string(), + // Encode a fake address as a second topic (just reuse the + // transfer symbol for shape — decode handles unknown gracefully). + TRANSFER_TOPIC_B64.to_string(), + ], + TRANSFER_VALUE_B64.to_string(), + ) + }) + .collect() +} + +/// Build a batch of pre-decoded `NewEvent`s ready for the enrichment or DB phase. +fn synthetic_new_events(n: usize) -> Vec { + (0..n) + .map(|i| NewEvent { + event_id: format!("bench-event-{i:08}-0000000000"), + contract_id: "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD2KM".into(), + ledger: 1_000_000 + i as i64, + ledger_closed_at: Utc::now(), + event_type: "contract".into(), + topics: vec![TRANSFER_TOPIC_B64.into()], + decoded_topics: vec![json!("transfer")], + event_name: Some("transfer".into()), + value: TRANSFER_VALUE_B64.into(), + decoded_value: json!("1000000"), + enriched: None, + tx_hash: format!("deadbeef{i:056x}"), + in_successful_call: true, + paging_token: format!("bench-event-{i:08}-0000000000"), + }) + .collect() +} + +/// Build a minimal `ContractSpec` with one event entry for `transfer`. +fn minimal_contract_spec() -> ContractSpec { + use stellar_xdr::curr::{ + ScSpecEventDataFormat, ScSpecEventParamLocationV0, ScSpecEventParamV0, ScSpecEventV0, + StringM, VecM, + }; + + let params: VecM = vec![ScSpecEventParamV0 { + doc: "".try_into().unwrap(), + name: StringM::try_from("amount").unwrap(), + type_: ScSpecTypeDef::I128, + location: ScSpecEventParamLocationV0::Data, + }] + .try_into() + .unwrap(); + + let entry = ScSpecEntry::EventV0(ScSpecEventV0 { + doc: "".try_into().unwrap(), + lib: "".try_into().unwrap(), + name: ScSymbol("transfer".try_into().unwrap()), + prefix_topics: vec![ScSymbol("transfer".try_into().unwrap())] + .try_into() + .unwrap(), + params, + data_format: ScSpecEventDataFormat::SingleValue, + }); + let bytes = entry.to_xdr(Limits::none()).unwrap(); + ContractSpec::from_spec_xdr(&bytes).unwrap_or_default() +} + +// ── Phase 1: XDR decode ─────────────────────────────────────────────────────── + +fn bench_xdr_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("xdr_decode"); + + for &batch in &[1usize, 100, 1_000, 10_000] { + group.throughput(Throughput::Elements(batch as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(batch), + &batch, + |b, &n| { + let raw = synthetic_raw_events(n); + b.iter(|| { + // Replicate what `convert::to_new_event` does for XDR. + let mut results = Vec::with_capacity(n); + for (topics, value) in &raw { + let decoded_topics = xdr::decode_topics(topics); + let decoded_value = xdr::decode_scval_base64(value); + let event_name = topics + .first() + .and_then(|t| xdr::event_name_from_topic(t)); + results.push((decoded_topics, decoded_value, event_name)); + } + results + }); + }, + ); + } + + group.finish(); +} + +// ── Phase 2: Enrichment ─────────────────────────────────────────────────────── + +fn bench_enrichment(c: &mut Criterion) { + let mut group = c.benchmark_group("enrichment"); + let spec = minimal_contract_spec(); + + for &batch in &[1usize, 100, 1_000, 10_000] { + group.throughput(Throughput::Elements(batch as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(batch), + &batch, + |b, &n| { + let events = synthetic_new_events(n); + b.iter(|| { + // Replicate what `convert::to_new_event` does for enrichment. + let mut enriched_count = 0usize; + for ev in &events { + if let Some(name) = &ev.event_name { + if spec + .enrich_event(name, &ev.decoded_topics, &ev.decoded_value) + .is_some() + { + enriched_count += 1; + } + } + } + enriched_count + }); + }, + ); + } + + group.finish(); +} + +// ── Phase 3: Database insert ────────────────────────────────────────────────── +// +// This phase requires a real Postgres database. Set TEST_DATABASE_URL to run it; +// otherwise the benchmark group is empty and criterion prints a warning. + +fn bench_db_insert(c: &mut Criterion) { + let db_url = match std::env::var("TEST_DATABASE_URL") { + Ok(u) => u, + Err(_) => { + eprintln!( + "\n[bench_indexer] db_insert phase SKIPPED — set TEST_DATABASE_URL to enable it.\n" + ); + return; + } + }; + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime"); + + // Set up a fresh schema for this benchmark run so inserts don't accumulate. + let pool = rt.block_on(async { + use sqlx::postgres::PgPoolOptions; + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&db_url) + .await + .expect("connect to TEST_DATABASE_URL"); + sqlx::migrate!("../../migrations") + .run(&pool) + .await + .expect("run migrations"); + pool + }); + + let mut group = c.benchmark_group("db_insert"); + + for &batch in &[10usize, 100, 500, 1_000] { + group.throughput(Throughput::Elements(batch as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(batch), + &batch, + |b, &n| { + b.to_async(&rt).iter_batched( + // Setup: generate a fresh set of events with unique IDs. + || synthetic_new_events(n), + // Routine: insert the batch and clean up. + |events| { + let pool = pool.clone(); + async move { + // Inline the core of `store::insert_events` to avoid + // depending on the private indexer module. + let event_ids: Vec<_> = + events.iter().map(|e| e.event_id.clone()).collect(); + let contract_ids: Vec<_> = + events.iter().map(|e| e.contract_id.clone()).collect(); + let ledgers: Vec = + events.iter().map(|e| e.ledger).collect(); + let closed_ats: Vec<_> = + events.iter().map(|e| e.ledger_closed_at).collect(); + let event_types: Vec<_> = + events.iter().map(|e| e.event_type.clone()).collect(); + let topics_json: Vec = events + .iter() + .map(|e| serde_json::to_string(&e.topics).unwrap()) + .collect(); + let decoded_topics_json: Vec = events + .iter() + .map(|e| serde_json::to_string(&e.decoded_topics).unwrap()) + .collect(); + let event_names: Vec> = + events.iter().map(|e| e.event_name.clone()).collect(); + let values: Vec<_> = + events.iter().map(|e| e.value.clone()).collect(); + let decoded_values_json: Vec = events + .iter() + .map(|e| serde_json::to_string(&e.decoded_value).unwrap()) + .collect(); + let enriched_json: Vec> = events + .iter() + .map(|e| { + e.enriched + .as_ref() + .map(|v| serde_json::to_string(v).unwrap()) + }) + .collect(); + let tx_hashes: Vec<_> = + events.iter().map(|e| e.tx_hash.clone()).collect(); + let in_successful_calls: Vec = + events.iter().map(|e| e.in_successful_call).collect(); + let paging_tokens: Vec<_> = + events.iter().map(|e| e.paging_token.clone()).collect(); + + sqlx::query( + "INSERT INTO events ( + event_id, contract_id, ledger, ledger_closed_at, event_type, + topics, decoded_topics, event_name, value, decoded_value, + enriched, tx_hash, in_successful_call, paging_token + ) + SELECT + event_id, contract_id, ledger, ledger_closed_at, event_type, + topics::jsonb, decoded_topics::jsonb, event_name, value, + decoded_value::jsonb, enriched::jsonb, tx_hash, + in_successful_call, paging_token + FROM UNNEST( + $1::text[], $2::text[], $3::bigint[], $4::timestamptz[], + $5::text[], $6::text[], $7::text[], $8::text[], $9::text[], + $10::text[], $11::text[], $12::text[], $13::bool[], $14::text[] + ) AS t( + event_id, contract_id, ledger, ledger_closed_at, event_type, + topics, decoded_topics, event_name, value, decoded_value, + enriched, tx_hash, in_successful_call, paging_token + ) + ON CONFLICT (event_id) DO NOTHING", + ) + .bind(&event_ids) + .bind(&contract_ids) + .bind(&ledgers) + .bind(&closed_ats) + .bind(&event_types) + .bind(&topics_json) + .bind(&decoded_topics_json) + .bind(&event_names) + .bind(&values) + .bind(&decoded_values_json) + .bind(&enriched_json) + .bind(&tx_hashes) + .bind(&in_successful_calls) + .bind(&paging_tokens) + .execute(&pool) + .await + .expect("insert_events"); + + // Clean up so subsequent iterations start fresh. + sqlx::query("DELETE FROM events") + .execute(&pool) + .await + .expect("cleanup"); + } + }, + BatchSize::PerIteration, + ); + }, + ); + } + + group.finish(); + rt.block_on(pool.close()); +} + +criterion_group!(benches, bench_xdr_decode, bench_enrichment, bench_db_insert); +criterion_main!(benches); diff --git a/crates/lumenqraph-indexer/src/backfill.rs b/crates/lumenqraph-indexer/src/backfill.rs index 720bd20..e179b97 100644 --- a/crates/lumenqraph-indexer/src/backfill.rs +++ b/crates/lumenqraph-indexer/src/backfill.rs @@ -184,7 +184,7 @@ mod tests { let rpc = RpcClient::new(&rpc_url, 30); let config = test_config(&rpc_url, 2); - let specs = SpecCache::new(2000); + let specs = SpecCache::new(2000, 4); let (inserted, _) = fetch_and_store(&pool, &rpc, &config, &specs, 500, 1000) .await @@ -233,7 +233,7 @@ mod tests { let config = test_config(&rpc_url, 2); // First run: all three events are new. - let specs = SpecCache::new(2000); + let specs = SpecCache::new(2000, 4); let (first, _) = fetch_and_store(&pool, &rpc, &config, &specs, 500, 1000) .await .expect("first run"); @@ -279,7 +279,7 @@ mod tests { let rpc = RpcClient::new(&rpc_url, 30); let config = test_config(&rpc_url, 2); - let specs = SpecCache::new(2000); + let specs = SpecCache::new(2000, 4); let (inserted, _) = fetch_and_store(&pool, &rpc, &config, &specs, 500, 1000) .await @@ -294,6 +294,57 @@ mod tests { assert_eq!(count, 2); } + /// After a simulated mid-backfill failure the cursor reflects the last + /// *successfully* stored page, not the start or the live tip. + /// + /// We verify this by running `fetch_and_store` for page 1 only (simulating + /// a two-page backfill where page 2 never starts), writing the cursor as + /// `backfill::run` now does after each page, then asserting the persisted + /// cursor equals the last ledger from page 1. + #[tokio::test] + #[ignore = "needs postgres"] + async fn backfill_cursor_checkpointed_per_page() { + let pool = fixture().await; + let tip = 1000i64; + + // Serve only one page (page 2 would need a second request, which the + // mock never receives because we stop after the first page). + let rpc_url = spawn_mock_rpc( + tip, + vec![MockPage { + cursor_in: None, + events: vec![make_event("e1", 500), make_event("e2", 501)], + // Return a cursor that signals "there is more" — but we won't + // request it (simulating an error mid-backfill). + cursor_out: Some("page2".into()), + }], + ) + .await; + + let rpc = RpcClient::new(&rpc_url, 30); + let config = test_config(&rpc_url, 2); + let specs = SpecCache::new(2000); + + // Simulate what backfill::run does: fetch page 1, store it, checkpoint. + let (inserted, _) = fetch_and_store(&pool, &rpc, &config, &specs, 500, tip) + .await + .expect("fetch_and_store page 1"); + + // Checkpoint after this page (ledger 501 is the max from page 1). + let page_max_ledger = 501i64; + cursor::write_progress(&pool, page_max_ledger, tip, inserted) + .await + .unwrap(); + + // The cursor must reflect page 1's last ledger, not 0 and not `tip`. + let last = cursor::read_last_processed(&pool).await.unwrap(); + assert_eq!( + last, + Some(page_max_ledger), + "cursor must be checkpointed to the last page's max ledger after a partial backfill" + ); + } + /// `backfill::run` clamps `from_ledger` to the oldest ledger the RPC still /// serves. This is a pure computation test — no DB or network needed. #[test] @@ -334,10 +385,23 @@ pub async fn run( ) -> anyhow::Result<()> { let tip = rpc.get_latest_ledger().await?; let oldest = tip - poller::max_lookback(); - let start = from_ledger.max(oldest).max(1); - if start > from_ledger { + + // If the caller passes 0 (or no explicit --from), try to resume from the + // last persisted cursor so an interrupted backfill picks up where it left + // off. A non-zero explicit from_ledger always wins. + let resume_from = if from_ledger == 0 { + cursor::read_last_processed(&pool) + .await? + .map(|l| l + 1) + .unwrap_or(0) + } else { + from_ledger + }; + + let start = resume_from.max(oldest).max(1); + if start > resume_from && resume_from > 0 { warn!( - requested = from_ledger, + requested = resume_from, clamped_to = start, "backfill start is older than RPC retention; clamping" ); @@ -345,8 +409,63 @@ pub async fn run( info!(from = start, to = tip, "starting backfill"); let specs = SpecCache::new(config.spec_cache_max_entries); - let (inserted, _) = fetch_and_store(&pool, &rpc, &config, &specs, start, tip).await?; - cursor::write_progress(&pool, tip, tip, inserted).await?; - info!(inserted, up_to_ledger = tip, "backfill complete"); + + // Drive the paging loop here so we can checkpoint the cursor after every + // successfully stored page. An interrupted backfill can be resumed by + // re-running with from_ledger = 0 (the default): the code above will read + // the persisted cursor and continue from the last completed page. + let mut cursor_token: Option = None; + let mut total_inserted = 0u64; + let mut last_page_ledger = start; + + loop { + let page = rpc + .get_events( + Some(start), + &config.contract_ids, + cursor_token.clone(), + config.page_size, + ) + .await?; + + let page_len = page.events.len(); + let page_max_ledger = page + .events + .iter() + .map(|e| e.ledger) + .max() + .unwrap_or(last_page_ledger); + + // Build and store this page's events. + let mut batch = Vec::with_capacity(page_len); + for ev in &page.events { + let spec = specs.get(&pool, &rpc, &ev.contract_id, ev.ledger).await; + batch.push(crate::convert::to_new_event(ev, spec.as_deref())); + } + let inserted = crate::store::insert_events(&pool, &batch).await?; + total_inserted += inserted; + + // ── checkpoint after each successfully stored page ────────────────── + // Writing the cursor here means a crash or Ctrl-C on the *next* page + // leaves the cursor pointing at the end of the last page we finished, + // so the backfill is resumable with no ledger range lost. + last_page_ledger = page_max_ledger; + cursor::write_progress(&pool, last_page_ledger, tip, inserted).await?; + info!( + inserted, + page_max_ledger, + total_inserted, + "backfill page complete (cursor checkpointed)" + ); + + cursor_token = page.cursor; + if page_len < config.page_size as usize || cursor_token.is_none() { + break; + } + } + + // Final cursor advance to the tip so the live poller starts from there. + cursor::write_progress(&pool, tip, tip, 0).await?; + info!(total_inserted, up_to_ledger = tip, "backfill complete"); Ok(()) } diff --git a/crates/lumenqraph-indexer/src/config.rs b/crates/lumenqraph-indexer/src/config.rs index b6c7ce6..a7f9f93 100644 --- a/crates/lumenqraph-indexer/src/config.rs +++ b/crates/lumenqraph-indexer/src/config.rs @@ -87,47 +87,23 @@ pub struct Config { /// indexing all contracts. Evicted entries are re-fetched from the database /// on next miss. Default: 2000. pub spec_cache_max_entries: usize, + /// After this many consecutive poll failures the poller enters a degraded + /// state: it sleeps for `degraded_poll_interval_secs` instead of the + /// normal backoff and emits an ERROR-level log. The counter resets on the + /// first successful cycle. 0 = never enter degraded state (disabled). + /// Default: 20. + pub max_consecutive_errors: u32, + /// Sleep interval (seconds) used while the circuit breaker is open (i.e. + /// the poller is in degraded state). Default: 300 (5 minutes). + pub degraded_poll_interval_secs: u64, } impl Config { pub fn from_env() -> anyhow::Result { - let contract_ids: Vec = std::env::var("CONTRACT_IDS") - .unwrap_or_default() - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - - // Validate CONTRACT_IDS as C-strkeys (Soroban contract addresses). - for id in &contract_ids { - if !lumenqraph_core::is_valid_contract_id(id) { - return Err(anyhow::anyhow!( - "invalid CONTRACT_ID {}: expected a C… strkey (Soroban contract address)", - id - )); - } - } - - // Validate the CONTRACT_IDS count against the getEvents RPC protocol limit: - // at most 5 filters × 5 IDs per filter = 25 IDs total. Checking this at - // startup produces a clear, actionable error message instead of a cryptic - // runtime failure on the first poll cycle. - const MAX_IDS_PER_FILTER: usize = 5; - const MAX_FILTERS: usize = 5; - const MAX_CONTRACT_IDS: usize = MAX_IDS_PER_FILTER * MAX_FILTERS; - if contract_ids.len() > MAX_CONTRACT_IDS { - return Err(anyhow::anyhow!( - "CONTRACT_IDS contains {} entries, but getEvents supports at most {} \ - contract IDs ({} filters × {} IDs per filter). \ - Remove {} contract IDs, or run multiple indexer instances each \ - covering a different subset.", - contract_ids.len(), - MAX_CONTRACT_IDS, - MAX_FILTERS, - MAX_IDS_PER_FILTER, - contract_ids.len() - MAX_CONTRACT_IDS, - )); - } + let contract_ids: Vec = lumenqraph_core::parse_contract_ids( + &std::env::var("CONTRACT_IDS").unwrap_or_default(), + ) + .map_err(|e| anyhow::anyhow!("{e}"))?; // Parse numeric config with validation. let poll_interval_secs = env_parse("POLL_INTERVAL_SECS", 5)?; @@ -138,6 +114,9 @@ impl Config { let reorg_overlap_ledgers = env_parse("REORG_OVERLAP_LEDGERS", 0)?; let rpc_timeout_secs = env_parse("RPC_TIMEOUT_SECS", 30u64)?; let spec_cache_max_entries = env_parse("SPEC_CACHE_MAX_ENTRIES", 2000usize)?; + let spec_fetch_concurrency = env_parse("SPEC_FETCH_CONCURRENCY", 4usize)?; + let database_max_connections = env_parse("DATABASE_MAX_CONNECTIONS", 10u32)?; + let database_min_connections = env_parse("DATABASE_MIN_CONNECTIONS", 0u32)?; // Validate and clamp PAGE_SIZE to RPC documented bounds (1–10000). let page_size = clamp_with_warning("PAGE_SIZE", page_size, 1, 10000); @@ -190,6 +169,23 @@ impl Config { // Validate SPEC_CACHE_MAX_ENTRIES minimum (must be at least 1). let spec_cache_max_entries = clamp_with_warning("SPEC_CACHE_MAX_ENTRIES", spec_cache_max_entries, 1, usize::MAX); + // Validate SPEC_FETCH_CONCURRENCY minimum (must be at least 1). + let spec_fetch_concurrency = clamp_with_warning("SPEC_FETCH_CONCURRENCY", spec_fetch_concurrency, 1, usize::MAX); + + // Validate DATABASE_MAX_CONNECTIONS minimum (must be at least 1). + let database_max_connections = clamp_with_warning("DATABASE_MAX_CONNECTIONS", database_max_connections, 1, u32::MAX); + + // Validate DATABASE_MIN_CONNECTIONS (must be <= max). + if database_min_connections > database_max_connections { + tracing::warn!( + requested_min = database_min_connections, + max = database_max_connections, + clamped_min = database_max_connections, + "DATABASE_MIN_CONNECTIONS cannot exceed DATABASE_MAX_CONNECTIONS; clamping to max" + ); + } + let database_min_connections = database_min_connections.min(database_max_connections); + // Parse ENRICHMENT_WARN_THRESHOLD (0.0-1.0, default 0.5). let enrichment_warn_threshold: f64 = env_parse("ENRICHMENT_WARN_THRESHOLD", 0.5)?; let enrichment_warn_threshold = if enrichment_warn_threshold < 0.0 || enrichment_warn_threshold > 1.0 { @@ -254,6 +250,9 @@ impl Config { .transpose()? .unwrap_or_default(); + let max_consecutive_errors = env_parse("MAX_CONSECUTIVE_ERRORS", 20u32)?; + let degraded_poll_interval_secs = env_parse("DEGRADED_POLL_INTERVAL_SECS", 300u64)?; + Ok(Self { database_url: env("DATABASE_URL")?, rpc_url: env("RPC_URL")?, @@ -277,6 +276,8 @@ impl Config { enrichment_warn_threshold, key_templates, spec_cache_max_entries, + max_consecutive_errors, + degraded_poll_interval_secs, }) } } diff --git a/crates/lumenqraph-indexer/src/cursor.rs b/crates/lumenqraph-indexer/src/cursor.rs index af12491..fe9d32e 100644 --- a/crates/lumenqraph-indexer/src/cursor.rs +++ b/crates/lumenqraph-indexer/src/cursor.rs @@ -1,7 +1,9 @@ //! The single-row indexer status (id = 1): ledger cursor plus health counters -//! that `/health` and `/metrics` read back. +//! that `/health` and `/metrics` read back. Uses optimistic locking (version column) +//! to detect and reject concurrent writer instances. use sqlx::PgPool; +use tracing::warn; /// Last fully-processed ledger, if the index has started. pub async fn read_last_processed(pool: &PgPool) -> anyhow::Result> { @@ -12,29 +14,56 @@ pub async fn read_last_processed(pool: &PgPool) -> anyhow::Result> { Ok(row.map(|r| r.0)) } +/// Read the current version for optimistic locking. +async fn read_version(pool: &PgPool) -> anyhow::Result { + let row: (i64,) = sqlx::query_as( + "SELECT COALESCE(version, 0) FROM indexer_cursor WHERE id = 1" + ) + .fetch_one(pool) + .await?; + Ok(row.0) +} + /// Advance the cursor and record the observed chain tip + how many events were -/// newly ingested this cycle. +/// newly ingested this cycle. Uses optimistic locking to detect concurrent writers. pub async fn write_progress( pool: &PgPool, last_processed: i64, chain_tip: i64, ingested_delta: u64, ) -> anyhow::Result<()> { - sqlx::query( - "INSERT INTO indexer_cursor - (id, last_processed_ledger, chain_tip_ledger, events_ingested_total, updated_at) - VALUES (1, $1, $2, $3, now()) - ON CONFLICT (id) DO UPDATE SET - last_processed_ledger = EXCLUDED.last_processed_ledger, - chain_tip_ledger = EXCLUDED.chain_tip_ledger, - events_ingested_total = indexer_cursor.events_ingested_total + $3, - updated_at = now()", + // Read current version for optimistic locking + let current_version = read_version(pool).await?; + + // Attempt update with version check; increment version on success + let rows_affected = sqlx::query( + "UPDATE indexer_cursor + SET last_processed_ledger = $1, + chain_tip_ledger = $2, + events_ingested_total = events_ingested_total + $3, + version = $4, + updated_at = now() + WHERE id = 1 AND version = $5", ) .bind(last_processed) .bind(chain_tip) .bind(ingested_delta as i64) + .bind(current_version + 1) + .bind(current_version) .execute(pool) - .await?; + .await? + .rows_affected(); + + if rows_affected == 0 { + warn!( + "cursor update failed: version mismatch (expected {}, cursor may have been updated by another instance)", + current_version + ); + return Err(anyhow::anyhow!( + "concurrent writer detected: version mismatch on cursor update" + )); + } + Ok(()) } @@ -111,3 +140,20 @@ pub async fn track_rpc_call( .await?; Ok(()) } + +/// Set the current consecutive-error count for the circuit-breaker Prometheus +/// gauge (`lumenqraph_consecutive_errors`). Called on every failure increment +/// and reset to 0 on the first successful cycle after a run of failures. +pub async fn set_consecutive_errors(pool: &PgPool, count: u32) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO indexer_cursor (id, last_processed_ledger, consecutive_errors, updated_at) + VALUES (1, 0, $1, now()) + ON CONFLICT (id) DO UPDATE SET + consecutive_errors = $1, + updated_at = now()", + ) + .bind(count as i64) + .execute(pool) + .await?; + Ok(()) +} diff --git a/crates/lumenqraph-indexer/src/deep_backfill.rs b/crates/lumenqraph-indexer/src/deep_backfill.rs index fd816f5..9d742f2 100644 --- a/crates/lumenqraph-indexer/src/deep_backfill.rs +++ b/crates/lumenqraph-indexer/src/deep_backfill.rs @@ -402,7 +402,7 @@ pub async fn run( to_ledger: Option, ) -> anyhow::Result<()> { let to = to_ledger.unwrap_or(i64::MAX); - let specs = SpecCache::new(config.spec_cache_max_entries); + let specs = SpecCache::new(config.spec_cache_max_entries, config.spec_fetch_concurrency); info!( from = from_ledger, diff --git a/crates/lumenqraph-indexer/src/main.rs b/crates/lumenqraph-indexer/src/main.rs index a15e6a9..3353756 100644 --- a/crates/lumenqraph-indexer/src/main.rs +++ b/crates/lumenqraph-indexer/src/main.rs @@ -28,7 +28,10 @@ mod rpc_client; mod specs; mod state; mod store; -#[cfg(test)] +// The end-to-end smoke test is gated behind the `smoke-tests` feature (as well +// as `#[ignore]`) so it is never compiled or run by a plain `cargo test`, +// including in offline CI. See CONTRIBUTING.md → "Smoke tests". +#[cfg(all(test, feature = "smoke-tests"))] mod smoke; use std::time::Duration; @@ -73,8 +76,8 @@ async fn main() -> anyhow::Result<()> { } let pool = PgPoolOptions::new() - .max_connections(env_parse_u32("DATABASE_MAX_CONNECTIONS", 5)) - .min_connections(env_parse_u32("DATABASE_MIN_CONNECTIONS", 1)) + .max_connections(config.database_max_connections) + .min_connections(config.database_min_connections) .acquire_timeout(Duration::from_secs(env_parse_u64( "DATABASE_ACQUIRE_TIMEOUT_SECS", 30, diff --git a/crates/lumenqraph-indexer/src/poller.rs b/crates/lumenqraph-indexer/src/poller.rs index 1fe82a9..dc278c7 100644 --- a/crates/lumenqraph-indexer/src/poller.rs +++ b/crates/lumenqraph-indexer/src/poller.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; use lumenqraph_core::NewEvent; use sqlx::PgPool; -use tracing::{debug, info, warn}; +use tracing::{debug, error, info, warn}; use crate::config::Config; use crate::convert::to_new_event; @@ -33,17 +33,30 @@ pub fn max_lookback() -> i64 { pub async fn run(pool: PgPool, rpc: RpcClient, config: Config) -> anyhow::Result<()> { let base_interval = Duration::from_secs(config.poll_interval_secs.max(1)); + let degraded_interval = Duration::from_secs(config.degraded_poll_interval_secs.max(1)); let mut backoff = base_interval; // One spec cache for the process lifetime: each contract's interface is // fetched and parsed once, then reused to enrich every event. - let specs = SpecCache::new(config.spec_cache_max_entries); + let specs = SpecCache::new(config.spec_cache_max_entries, config.spec_fetch_concurrency); // None => prune on the first cycle that reaches the tip, so a deployment // that switches retention on starts reclaiming immediately. let mut last_prune: Option = None; + // Circuit-breaker state: count consecutive poll failures. + let mut consecutive_errors: u32 = 0; loop { let sleep_for = match poll_once(&pool, &rpc, &config, &specs).await { Ok(processed_to) => { + // Success: reset both the backoff and the circuit-breaker counter. + if consecutive_errors > 0 { + info!( + consecutive_errors, + "poll cycle succeeded; resetting circuit breaker" + ); + consecutive_errors = 0; + // Record cleared state to the gauge. + let _ = cursor::set_consecutive_errors(&pool, 0).await; + } backoff = base_interval; if let Some(ledger) = processed_to { debug!(ledger, "cycle complete"); @@ -77,11 +90,31 @@ pub async fn run(pool: PgPool, rpc: RpcClient, config: Config) -> anyhow::Result base_interval } Err(e) => { - warn!(error = %e, backoff_secs = backoff.as_secs(), "poll cycle failed; backing off"); + consecutive_errors += 1; let _ = cursor::incr_errors(&pool).await; - let this = backoff; - backoff = (backoff * 2).min(Duration::from_secs(60)); - this + let _ = cursor::set_consecutive_errors(&pool, consecutive_errors).await; + + // Check if we should enter degraded / circuit-breaker state. + let circuit_open = config.max_consecutive_errors > 0 + && consecutive_errors >= config.max_consecutive_errors; + + if circuit_open { + error!( + error = %e, + consecutive_errors, + max_consecutive_errors = config.max_consecutive_errors, + degraded_interval_secs = config.degraded_poll_interval_secs, + "circuit breaker open: too many consecutive poll failures; \ + switching to degraded polling interval" + ); + backoff = degraded_interval; + degraded_interval + } else { + warn!(error = %e, backoff_secs = backoff.as_secs(), consecutive_errors, "poll cycle failed; backing off"); + let this = backoff; + backoff = (backoff * 2).min(Duration::from_secs(60)); + this + } } }; @@ -519,4 +552,52 @@ mod tests { "max_lookback() should export the RPC retention window" ); } + + // ── Circuit breaker logic ───────────────────────────────────────────── + + #[test] + fn circuit_breaker_opens_after_max_consecutive_errors() { + let max = 20u32; + // Simulate accumulating errors. + for count in 1..=max { + let circuit_open = max > 0 && count >= max; + if count < max { + assert!(!circuit_open, "circuit should stay closed at error {count}"); + } else { + assert!(circuit_open, "circuit should open at error {count}"); + } + } + } + + #[test] + fn circuit_breaker_disabled_when_max_is_zero() { + // max_consecutive_errors = 0 means the circuit breaker is disabled. + let max = 0u32; + let count = 1_000u32; + let circuit_open = max > 0 && count >= max; + assert!(!circuit_open, "circuit should never open when max = 0"); + } + + #[test] + fn circuit_breaker_resets_on_success() { + // After a successful cycle consecutive_errors is reset to 0. + let mut consecutive_errors: u32 = 25; + // Simulate success path. + if consecutive_errors > 0 { + consecutive_errors = 0; + } + assert_eq!(consecutive_errors, 0); + } + + #[test] + fn degraded_interval_used_when_circuit_open() { + let base = Duration::from_secs(5); + let degraded = Duration::from_secs(300); + let max_consecutive_errors = 20u32; + let consecutive_errors = 20u32; + + let circuit_open = max_consecutive_errors > 0 && consecutive_errors >= max_consecutive_errors; + let sleep = if circuit_open { degraded } else { base }; + assert_eq!(sleep, degraded, "degraded interval should be used when circuit is open"); + } } diff --git a/crates/lumenqraph-indexer/src/reenrich.rs b/crates/lumenqraph-indexer/src/reenrich.rs index 419dd1d..497a3e4 100644 --- a/crates/lumenqraph-indexer/src/reenrich.rs +++ b/crates/lumenqraph-indexer/src/reenrich.rs @@ -5,6 +5,9 @@ //! This is a one-shot backfill pass that can be run manually or automatically //! when a spec is first successfully fetched for a contract with stored events. +use std::io::{self, IsTerminal}; +use std::time::Instant; + use lumenqraph_core::NewEvent; use sqlx::{PgPool, Row}; use tracing::{debug, info, warn}; @@ -18,14 +21,20 @@ use crate::specs::SpecCache; /// AND event_name IS NOT NULL, re-enrich them against the (now-cached) spec, /// and update the database. pub async fn run_reenrich(pool: PgPool, rpc: RpcClient, config: Config) -> anyhow::Result<()> { - let specs = SpecCache::new(config.spec_cache_max_entries); + let specs = SpecCache::new(config.spec_cache_max_entries, config.spec_fetch_concurrency); let mut processed = 0u64; let mut updated = 0u64; // Fetch events in batches to avoid loading the entire table into memory. const BATCH_SIZE: i32 = 1000; + const PROGRESS_INTERVAL: u64 = 10_000; let mut offset = 0i32; + let is_tty = io::stderr().is_terminal(); + let start_time = Instant::now(); + let mut last_progress_time = start_time; + let mut last_progress_count = 0u64; + loop { let rows = sqlx::query( "SELECT event_id, contract_id, decoded_topics, event_name, decoded_value @@ -52,6 +61,50 @@ pub async fn run_reenrich(pool: PgPool, rpc: RpcClient, config: Config) -> anyho processed += 1; + // Report progress every PROGRESS_INTERVAL events + if processed % PROGRESS_INTERVAL == 0 { + let elapsed = start_time.elapsed(); + let events_since_last = processed - last_progress_count; + let time_since_last = last_progress_time.elapsed(); + + if time_since_last.as_secs_f64() > 0.0 { + let throughput = events_since_last as f64 / time_since_last.as_secs_f64(); + let remaining = total_events.saturating_sub(processed); + let remaining_estimate = if throughput > 0.0 { + std::time::Duration::from_secs_f64(remaining as f64 / throughput) + } else { + std::time::Duration::ZERO + }; + + let elapsed_secs = elapsed.as_secs(); + let remaining_secs = remaining_estimate.as_secs(); + let remaining_fmt = if remaining_secs < 3600 { + format!("{:.0}m", remaining_secs as f64 / 60.0) + } else { + format!("{:.1}h", remaining_secs as f64 / 3600.0) + }; + + if is_tty { + eprintln!( + "Re-enriching… {processed:>10}/{total_events:<10} | {elapsed_secs:>5}s | {throughput:.0} evt/s | ~{remaining_fmt} remaining" + ); + } + + info!( + processed, + total_events, + updated, + elapsed_secs, + throughput = throughput as u64, + remaining_secs, + "re-enrichment progress" + ); + + last_progress_time = Instant::now(); + last_progress_count = processed; + } + } + // Re-parse the decoded data for enrichment. let decoded_topics: Vec = match serde_json::from_str(&decoded_topics_json) { diff --git a/crates/lumenqraph-indexer/src/rpc_client.rs b/crates/lumenqraph-indexer/src/rpc_client.rs index 76a2d13..3b34421 100644 --- a/crates/lumenqraph-indexer/src/rpc_client.rs +++ b/crates/lumenqraph-indexer/src/rpc_client.rs @@ -494,10 +494,21 @@ impl RpcClient { /// Reset and return the accumulated RPC metrics from this client instance. /// Used by the indexer to report metrics periodically. + /// + /// Memory ordering rationale: + /// - `fetch_add` on the counter paths uses `Relaxed` because counter + /// increments are independent — we only care about the final aggregate, + /// not any ordering relative to other memory operations. + /// - `swap(0, Acquire)` here ensures that all preceding `Relaxed` + /// `fetch_add` operations on *this thread* (and any thread that + /// synchronised with this one) are visible before the counters are + /// reset. This prevents a stale read where increments that happened + /// before the swap are not yet visible to the Prometheus reporter on + /// weakly-ordered architectures such as ARM. pub fn take_metrics(&self) -> (u64, u64, u64) { - let calls = self.calls_made.swap(0, std::sync::atomic::Ordering::Relaxed); - let errors = self.calls_failed.swap(0, std::sync::atomic::Ordering::Relaxed); - let errors_32001 = self.calls_failed_32001.swap(0, std::sync::atomic::Ordering::Relaxed); + let calls = self.calls_made.swap(0, std::sync::atomic::Ordering::Acquire); + let errors = self.calls_failed.swap(0, std::sync::atomic::Ordering::Acquire); + let errors_32001 = self.calls_failed_32001.swap(0, std::sync::atomic::Ordering::Acquire); (calls, errors, errors_32001) } diff --git a/crates/lumenqraph-indexer/src/smoke.rs b/crates/lumenqraph-indexer/src/smoke.rs index da20191..2685866 100644 --- a/crates/lumenqraph-indexer/src/smoke.rs +++ b/crates/lumenqraph-indexer/src/smoke.rs @@ -9,9 +9,15 @@ //! creates pending deliveries; a local HTTP sink receives and //! validates the signed POST. //! -//! Run with: +//! This module is gated behind the `smoke-tests` cargo feature *and* +//! `#[ignore]`, so a plain `cargo test` never compiles or runs it (offline CI +//! stays offline). Run it explicitly with: +//! //! TEST_DATABASE_URL=postgres://…/lumenqraph \ -//! cargo test -p lumenqraph-indexer smoke -- --ignored --test-threads=1 +//! cargo test -p lumenqraph-indexer --features smoke-tests smoke \ +//! -- --ignored --test-threads=1 +//! +//! or `make test-smoke`. See CONTRIBUTING.md → "Smoke tests". #[cfg(test)] mod tests { @@ -214,7 +220,7 @@ mod tests { let rpc = RpcClient::new(&rpc_url, 30); let config = test_config(&rpc_url, 2); - let specs = SpecCache::new(config.spec_cache_max_entries); + let specs = SpecCache::new(config.spec_cache_max_entries, config.spec_fetch_concurrency); let (inserted, _) = poller::fetch_and_store(&pool, &rpc, &config, &specs, 500, 1000) .await diff --git a/crates/lumenqraph-indexer/src/specs.rs b/crates/lumenqraph-indexer/src/specs.rs index 956c8f6..5d9cd94 100644 --- a/crates/lumenqraph-indexer/src/specs.rs +++ b/crates/lumenqraph-indexer/src/specs.rs @@ -23,6 +23,7 @@ use std::time::{Duration, Instant}; use lru::LruCache; use lumenqraph_core::{ContractSpec, SpecDiff}; use sqlx::PgPool; +use tokio::sync::Semaphore; use tracing::{debug, info, warn}; use crate::rpc_client::RpcClient; @@ -57,14 +58,16 @@ const FETCH_ERROR_TTL: Duration = Duration::from_secs(60); pub struct SpecCache { inner: Mutex>, + fetch_semaphore: Arc, } impl SpecCache { - pub fn new(max_entries: usize) -> Self { + pub fn new(max_entries: usize, concurrency: usize) -> Self { Self { inner: Mutex::new(LruCache::new( std::num::NonZeroUsize::new(max_entries).expect("max_entries must be > 0"), )), + fetch_semaphore: Arc::new(Semaphore::new(concurrency)), } } @@ -84,6 +87,7 @@ impl SpecCache { /// The spec for a contract, fetching+parsing+persisting on first use. /// Distinguishes transient failures (retryable) from permanent failures (SAC). + /// Concurrent fetches are bounded by the semaphore; already-cached lookups bypass it. pub async fn get( &self, pool: &PgPool, @@ -104,6 +108,8 @@ impl SpecCache { } } } + // Acquire semaphore permit before fetching — this limits concurrent fetches. + let _permit = self.fetch_semaphore.acquire().await.expect("semaphore acquire failed"); let (spec, wasm_hash, is_permanent) = load(pool, rpc, contract_id, ledger).await; let cached_spec = match spec { Some(s) => CachedSpec::Spec(s.clone()), diff --git a/crates/lumenqraph-mcp/src/main.rs b/crates/lumenqraph-mcp/src/main.rs index 9f3b362..8a9eaea 100644 --- a/crates/lumenqraph-mcp/src/main.rs +++ b/crates/lumenqraph-mcp/src/main.rs @@ -23,7 +23,7 @@ use anyhow::Context; use serde_json::{json, Value}; use sqlx::postgres::PgPoolOptions; use sqlx::PgPool; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::io::{AsyncBufReadExt, BufReader}; use tracing::info; use tracing_subscriber::{fmt, prelude::*, EnvFilter}; @@ -55,6 +55,13 @@ async fn main() -> anyhow::Result<()> { .and_then(|v| v.trim().parse().ok()) .unwrap_or(30); + // Validate CONTRACT_IDS at startup so a misconfigured address is caught + // immediately rather than silently ignored. + lumenqraph_core::parse_contract_ids( + &std::env::var("CONTRACT_IDS").unwrap_or_default(), + ) + .map_err(|e| anyhow::anyhow!("{e}"))?; + let pool = PgPoolOptions::new() .max_connections(5) .connect(&database_url) @@ -71,8 +78,18 @@ async fn main() -> anyhow::Result<()> { /// The stdio JSON-RPC loop: read a message per line, dispatch, write responses. async fn serve(state: State) -> anyhow::Result<()> { - let mut lines = BufReader::new(tokio::io::stdin()).lines(); - let mut stdout = tokio::io::stdout(); + serve_io(state, tokio::io::stdin(), tokio::io::stdout()).await +} + +/// Protocol loop over any `AsyncRead` / `AsyncWrite` pair (stdin/stdout in +/// production, in-memory duplex streams in tests). +pub(crate) async fn serve_io(state: State, reader: R, writer: W) -> anyhow::Result<()> +where + R: tokio::io::AsyncRead + Unpin, + W: tokio::io::AsyncWrite + Unpin, +{ + let mut lines = BufReader::new(reader).lines(); + let mut out = writer; while let Some(line) = lines.next_line().await? { if line.trim().is_empty() { continue; @@ -81,21 +98,25 @@ async fn serve(state: State) -> anyhow::Result<()> { Ok(v) => v, Err(e) => { let err = error_response(Value::Null, -32700, &format!("parse error: {e}")); - write(&mut stdout, &err).await?; + write_to(&mut out, &err).await?; continue; } }; if let Some(response) = handle(&state, msg).await { - write(&mut stdout, &response).await?; + write_to(&mut out, &response).await?; } } Ok(()) } -async fn write(stdout: &mut tokio::io::Stdout, value: &Value) -> anyhow::Result<()> { - stdout.write_all(value.to_string().as_bytes()).await?; - stdout.write_all(b"\n").await?; - stdout.flush().await?; +async fn write_to( + writer: &mut W, + value: &Value, +) -> anyhow::Result<()> { + use tokio::io::AsyncWriteExt as _; + writer.write_all(value.to_string().as_bytes()).await?; + writer.write_all(b"\n").await?; + writer.flush().await?; Ok(()) } @@ -518,3 +539,310 @@ mod tests { ); } } + +/// Integration tests for the JSON-RPC protocol layer (`serve_io`). +/// +/// These tests drive the MCP server through the full stdio round-trip using an +/// in-process `tokio::io::duplex` stream pair, covering: +/// - The initialize → tools/list → tools/call handshake +/// - Malformed JSON input (parse error -32700) +/// - Unknown method (method-not-found error -32601) +/// - Notifications (no response expected) +/// - Missing required tool argument (isError result) +/// +/// No real database or RPC server is required; the pool is created with +/// `connect_lazy` so no network calls are made before the tests exercise +/// validation paths. +#[cfg(test)] +mod protocol_tests { + use super::*; + + /// Build a `State` that does not need a live database. + fn lazy_state() -> State { + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy("postgres://test:test@localhost/test") + .expect("connect_lazy"); + State { + pool, + rpc: RpcClient::new("http://127.0.0.1:0", 30), + } + } + + /// Feed `input` (newline-delimited JSON-RPC messages) through `serve_io` + /// and return all response lines as parsed `serde_json::Value`s. + /// + /// Uses a simplex stream: we write all input into one half of a duplex, + /// close the write end (signalling EOF), run serve_io against it, then + /// collect all output from the output half of a second duplex. + async fn run_rpc(input: &str) -> Vec { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let state = lazy_state(); + + // Build the input side: a DuplexStream where we write the test messages + // then drop the write half to signal EOF. + let (mut write_half, read_half) = tokio::io::duplex(64 * 1024); + write_half.write_all(input.as_bytes()).await.expect("write input"); + drop(write_half); // EOF for the server reader + + // The output side: a DuplexStream where the server writes responses. + let (out_write_half, mut out_read_half) = tokio::io::duplex(64 * 1024); + + // Run serve_io to completion; it will exit when the reader hits EOF. + serve_io(state, read_half, out_write_half) + .await + .expect("serve_io should not fail"); + + // Read all bytes written by serve_io. + let mut buf = Vec::new(); + out_read_half.read_to_end(&mut buf).await.expect("read output"); + let output = String::from_utf8(buf).expect("utf8 output"); + + output + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).unwrap_or_else(|e| { + panic!("server produced invalid JSON: {e}\nline: {l}") + })) + .collect() + } + + // ── initialize handshake ────────────────────────────────────────────── + + #[tokio::test] + async fn initialize_round_trip() { + let msgs = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{}}} +"#; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 1, "exactly one response to initialize"); + let r = &responses[0]; + assert_eq!(r["id"], 1); + assert_eq!(r["result"]["protocolVersion"], "2024-11-05"); + assert_eq!(r["result"]["serverInfo"]["name"], "lumenqraph-mcp"); + assert!(r["result"]["capabilities"]["tools"].is_object()); + } + + // ── tools/list ──────────────────────────────────────────────────────── + + #[tokio::test] + async fn tools_list_round_trip() { + let msgs = r#"{"jsonrpc":"2.0","id":2,"method":"tools/list"} +"#; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 1); + let r = &responses[0]; + assert_eq!(r["id"], 2); + let tools = r["result"]["tools"].as_array().expect("tools array"); + assert!(!tools.is_empty(), "at least one tool must be declared"); + for tool in tools { + assert!(!tool["name"].as_str().unwrap_or("").is_empty()); + assert_eq!(tool["inputSchema"]["type"], "object"); + } + } + + // ── full initialize → tools/list → tools/call round-trip ───────────── + + #[tokio::test] + async fn full_handshake_round_trip() { + // Three messages: initialize, tools/list, and a tools/call that will + // fail with isError (no DB) but still produce a well-formed response. + let msgs = concat!( + r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{}}}"#, "\n", + r#"{"jsonrpc":"2.0","id":2,"method":"tools/list"}"#, "\n", + r#"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"list_contracts","arguments":{}}}"#, "\n", + ); + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 3, "one response per request"); + + assert_eq!(responses[0]["id"], 1, "init id"); + assert!(responses[0]["result"]["protocolVersion"].is_string()); + + assert_eq!(responses[1]["id"], 2, "tools/list id"); + assert!(responses[1]["result"]["tools"].is_array()); + + assert_eq!(responses[2]["id"], 3, "tools/call id"); + // Either a real result or an isError — both are valid without a DB. + let result = &responses[2]["result"]; + assert!(result.is_object(), "tools/call always produces a result object"); + } + + // ── error cases ─────────────────────────────────────────────────────── + + #[tokio::test] + async fn malformed_json_returns_parse_error() { + let msgs = "not valid json at all\n"; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 1); + let r = &responses[0]; + // id must be null/absent for a parse error (we can't know the request id). + assert_eq!(r["error"]["code"], -32700); + assert!( + r["error"]["message"].as_str().unwrap_or("").contains("parse error"), + "message should say parse error: {:?}", r["error"]["message"] + ); + } + + #[tokio::test] + async fn unknown_method_returns_method_not_found() { + let msgs = r#"{"jsonrpc":"2.0","id":9,"method":"no/such/method"} +"#; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 1); + let r = &responses[0]; + assert_eq!(r["id"], 9); + assert_eq!(r["error"]["code"], -32601); + assert!( + r["error"]["message"].as_str().unwrap_or("").contains("no/such/method"), + "error should mention the method: {:?}", r["error"]["message"] + ); + } + + #[tokio::test] + async fn notification_produces_no_response() { + // Notifications have no `id`; the server must not reply. + let msgs = r#"{"jsonrpc":"2.0","method":"notifications/initialized"} +"#; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 0, "notifications must not produce a response"); + } + + #[tokio::test] + async fn empty_lines_are_ignored() { + let msgs = "\n\n \n"; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 0, "blank lines produce no output"); + } + + #[tokio::test] + async fn ping_returns_empty_result() { + let msgs = r#"{"jsonrpc":"2.0","id":42,"method":"ping"} +"#; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 1); + let r = &responses[0]; + assert_eq!(r["id"], 42); + assert!(r["result"].is_object()); + assert!(r.get("error").is_none()); + } + + #[tokio::test] + async fn missing_required_arg_returns_is_error() { + // get_contract_interface requires contract_id; omitting it must produce + // isError: true (MCP convention — tool errors are results, not protocol errors). + let msgs = r#"{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"get_contract_interface","arguments":{}}} +"#; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 1); + let r = &responses[0]; + assert_eq!(r["id"], 5); + let result = &r["result"]; + assert_eq!(result["isError"], true, "missing arg must set isError"); + let text = result["content"][0]["text"].as_str().unwrap_or(""); + assert!( + text.contains("contract_id"), + "error text should mention 'contract_id': {text}" + ); + } + + #[tokio::test] + async fn unknown_tool_name_returns_is_error_via_protocol() { + let msgs = r#"{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"does_not_exist","arguments":{}}} +"#; + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 1); + let r = &responses[0]; + assert_eq!(r["id"], 7); + assert_eq!(r["result"]["isError"], true); + let text = r["result"]["content"][0]["text"].as_str().unwrap_or(""); + assert!( + text.contains("does_not_exist"), + "error should mention the unknown tool: {text}" + ); + } + + #[tokio::test] + async fn multiple_messages_get_independent_responses() { + // Two well-formed requests: both must produce a response, in order. + let msgs = concat!( + r#"{"jsonrpc":"2.0","id":10,"method":"ping"}"#, "\n", + r#"{"jsonrpc":"2.0","id":11,"method":"ping"}"#, "\n", + ); + let responses = run_rpc(msgs).await; + assert_eq!(responses.len(), 2); + assert_eq!(responses[0]["id"], 10); + assert_eq!(responses[1]["id"], 11); + } +} + +/// #219 — CONTRACT_IDS startup validation in lumenqraph-mcp. +/// +/// The MCP server calls `lumenqraph_core::parse_contract_ids` at startup and +/// propagates the error so the process refuses to start on a misconfigured +/// address. These tests exercise the same validation logic directly, without +/// needing a live Postgres connection or stdio pipe, to ensure the guard +/// never silently regresses. +#[cfg(test)] +mod contract_ids_startup_validation { + #[test] + fn rejects_g_strkey_account_address() { + // A G… strkey is a Stellar account address, not a Soroban contract. + // A G-strkey accidentally placed in CONTRACT_IDS must be caught here. + let raw = "GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "error should mention invalid CONTRACT_ID: {err}" + ); + assert!( + err.contains("GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"), + "error should quote the bad id: {err}" + ); + } + + #[test] + fn rejects_garbage_string() { + let raw = "not-a-contract-id"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "garbage string should be rejected: {err}" + ); + } + + #[test] + fn rejects_too_many_contract_ids() { + // getEvents supports at most 25 IDs; the parser enforces this. + let single = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"; + let raw = std::iter::repeat(single).take(26).collect::>().join(","); + let err = lumenqraph_core::parse_contract_ids(&raw).unwrap_err(); + assert!( + err.contains("26"), + "error should mention the count 26: {err}" + ); + } + + #[test] + fn accepts_empty_string() { + // Empty CONTRACT_IDS means "index all" — must not be an error. + let ids = lumenqraph_core::parse_contract_ids("").unwrap(); + assert!(ids.is_empty(), "empty string should yield zero IDs"); + } + + #[test] + fn accepts_valid_c_strkey() { + let raw = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"; + let ids = lumenqraph_core::parse_contract_ids(raw).unwrap(); + assert_eq!(ids.len(), 1); + assert_eq!(ids[0], raw); + } + + #[test] + fn mixed_valid_and_invalid_is_rejected() { + let raw = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC,GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "a G-strkey mixed with a valid C-strkey should be rejected: {err}" + ); + } +} diff --git a/crates/lumenqraph-webhooks/src/config.rs b/crates/lumenqraph-webhooks/src/config.rs index d3ade1c..f141410 100644 --- a/crates/lumenqraph-webhooks/src/config.rs +++ b/crates/lumenqraph-webhooks/src/config.rs @@ -14,10 +14,29 @@ pub struct Config { pub max_concurrent_per_host: usize, pub max_concurrent_deliveries: usize, pub failure_threshold: i32, + /// The `pgp_sym_encrypt` / `pgp_sym_decrypt` key used for the webhook + /// shared secrets. Read once at startup and never falls back to a default, + /// so a missing key is a hard startup failure rather than a silent + /// security regression. + pub encryption_key: String, } impl Config { pub fn from_env() -> anyhow::Result { + let encryption_key = std::env::var("WEBHOOK_ENCRYPTION_KEY") + .map_err(|_| anyhow::anyhow!( + "WEBHOOK_ENCRYPTION_KEY must be set \ + (generate with: openssl rand -hex 32). \ + The default test key provides no security and must not be \ + used in production." + ))?; + if encryption_key.trim().is_empty() { + anyhow::bail!( + "WEBHOOK_ENCRYPTION_KEY is set but empty; \ + generate a key with: openssl rand -hex 32" + ); + } + Ok(Self { database_url: std::env::var("DATABASE_URL").context("missing DATABASE_URL")?, tick_secs: parse("WEBHOOK_TICK_SECS", 3), @@ -28,6 +47,7 @@ impl Config { max_concurrent_per_host: parse("WEBHOOK_MAX_CONCURRENT_PER_HOST", 5), max_concurrent_deliveries: parse("WEBHOOK_MAX_CONCURRENT_DELIVERIES", 100), failure_threshold: parse("WEBHOOK_FAILURE_THRESHOLD", 10), + encryption_key, }) } @@ -46,3 +66,33 @@ fn parse(key: &str, default: T) -> T { .and_then(|v| v.parse().ok()) .unwrap_or(default) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_env_fails_fast_without_encryption_key() { + // Make sure DATABASE_URL is present (required) but WEBHOOK_ENCRYPTION_KEY + // is absent — Config::from_env() must return an error. + // We use a controlled sub-environment by temporarily unsetting the var. + // This is a best-effort unit test; the real guard is the integration. + let result = { + // Temporarily remove the key from this process's env if it's set. + let original = std::env::var("WEBHOOK_ENCRYPTION_KEY").ok(); + unsafe { std::env::remove_var("WEBHOOK_ENCRYPTION_KEY"); } + let r = Config::from_env(); + // Restore. + if let Some(val) = original { + unsafe { std::env::set_var("WEBHOOK_ENCRYPTION_KEY", val); } + } + r + }; + // The error must mention WEBHOOK_ENCRYPTION_KEY. + let err = result.unwrap_err(); + assert!( + err.to_string().contains("WEBHOOK_ENCRYPTION_KEY"), + "error should mention the missing var: {err}" + ); + } +} diff --git a/crates/lumenqraph-webhooks/src/dispatcher.rs b/crates/lumenqraph-webhooks/src/dispatcher.rs index e97c14c..8500feb 100644 --- a/crates/lumenqraph-webhooks/src/dispatcher.rs +++ b/crates/lumenqraph-webhooks/src/dispatcher.rs @@ -19,6 +19,7 @@ use sha2::Sha256; use sqlx::types::Json; use sqlx::PgPool; use std::collections::HashMap; +use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::Arc; use tokio::sync::Semaphore; use tracing::{debug, info, warn}; @@ -26,6 +27,7 @@ use url::Url; use crate::config::Config; use futures::stream::{self, StreamExt}; +use lumenqraph_core::url_validation::validate_webhook_url_at_delivery; type HmacSha256 = Hmac; @@ -34,6 +36,27 @@ type HmacSha256 = Hmac; /// version that sent them. const USER_AGENT: &str = concat!("lumenqraph-webhooks/", env!("CARGO_PKG_VERSION")); +/// Last observed count of `pending` rows in `webhook_deliveries`, refreshed once +/// per dispatcher tick by [`refresh_pending_gauge`] and read by the `/metrics` +/// endpoint (`lumenqraph_webhooks_pending_deliveries`). +/// +/// Enqueue and deliver counts only describe what moved *this* tick; they go +/// quiet when the dispatcher is starved by a slow or unresponsive subscriber +/// even though the backlog is growing. This gauge makes that backlog the one +/// number an operator can alert on. +pub static PENDING_DELIVERIES: AtomicI64 = AtomicI64::new(0); + +/// Refresh [`PENDING_DELIVERIES`] from the database. Called once per tick by the +/// service loop; returns the value it stored so the caller can log it. +pub async fn refresh_pending_gauge(pool: &PgPool) -> anyhow::Result { + let pending: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM webhook_deliveries WHERE status = 'pending'") + .fetch_one(pool) + .await?; + PENDING_DELIVERIES.store(pending, Ordering::Relaxed); + Ok(pending) +} + /// Enqueue deliveries for everything new in both streams. Returns how many /// delivery rows were created. pub async fn enqueue(pool: &PgPool, batch: i64) -> anyhow::Result { @@ -175,10 +198,7 @@ struct DueDelivery { /// keep their long-standing shape (the bare event row); upgrade payloads are /// tagged, since they're a new shape and a consumer receiving one should be able /// to tell what it is. -async fn fetch_due(pool: &PgPool, batch: i64) -> anyhow::Result> { - let encryption_key = std::env::var("WEBHOOK_ENCRYPTION_KEY") - .unwrap_or_else(|_| "default-key-for-testing".to_string()); - +async fn fetch_due(pool: &PgPool, batch: i64, encryption_key: &str) -> anyhow::Result> { let rows: Vec<(i64, String, i32, String, String, Json)> = sqlx::query_as( "SELECT d.id, s.id, d.attempts, s.url, pgp_sym_decrypt(s.encrypted_secret, $1), @@ -226,7 +246,7 @@ pub async fn deliver( http: &reqwest::Client, config: &Config, ) -> anyhow::Result<(u64, u64)> { - let deliveries = fetch_due(pool, config.batch_size).await?; + let deliveries = fetch_due(pool, config.batch_size, &config.encryption_key).await?; if deliveries.is_empty() { return Ok((0, 0)); } @@ -316,6 +336,12 @@ fn extract_host(url: &str) -> String { } async fn send(http: &reqwest::Client, d: &DueDelivery, config: &Config) -> anyhow::Result<()> { + // Re-validate URL at delivery time to prevent DNS rebinding attacks. + // This ensures the hostname still resolves to a public address even if the + // DNS record changed since registration. + validate_webhook_url_at_delivery(&d.url).await + .map_err(|e| anyhow::anyhow!("URL validation failed at delivery: {}", e))?; + let body = serde_json::to_vec(&d.payload.0)?; let timestamp = Utc::now().to_rfc3339(); @@ -565,7 +591,7 @@ mod tests { "only the upgrade (v2) enqueues; v1 is a baseline, not a change" ); - let due = fetch_due(&pool, 100).await.unwrap(); + let due = fetch_due(&pool, 100, "test-key").await.unwrap(); assert_eq!(due.len(), 1); let payload = &due[0].payload.0; assert_eq!(payload["type"], "contract.upgraded"); @@ -622,7 +648,7 @@ mod tests { assert_eq!(enqueue(&pool, 100).await.unwrap(), 1); // The watermark has advanced, so a second pass finds nothing new. assert_eq!(enqueue(&pool, 100).await.unwrap(), 0); - assert_eq!(fetch_due(&pool, 100).await.unwrap().len(), 1); + assert_eq!(fetch_due(&pool, 100, "test-key").await.unwrap().len(), 1); } #[tokio::test] @@ -638,7 +664,7 @@ mod tests { assert_eq!(enqueue(&pool, 100).await.unwrap(), 5); // Simulate failures for all 5 deliveries. - let due = fetch_due(&pool, 100).await.unwrap(); + let due = fetch_due(&pool, 100, "test-key").await.unwrap(); assert_eq!(due.len(), 5); for d in due { diff --git a/crates/lumenqraph-webhooks/src/main.rs b/crates/lumenqraph-webhooks/src/main.rs index 618ca64..029aebe 100644 --- a/crates/lumenqraph-webhooks/src/main.rs +++ b/crates/lumenqraph-webhooks/src/main.rs @@ -66,14 +66,15 @@ async fn main() -> anyhow::Result<()> { .with(fmt::layer()) .init(); - // Validate webhook encryption key is set for production security - if std::env::var("WEBHOOK_ENCRYPTION_KEY").is_err() { - anyhow::bail!( - "WEBHOOK_ENCRYPTION_KEY must be set (generate with: openssl rand -hex 32). \ - The default test key provides no security and must not be used in production." - ); - } - + // Validate CONTRACT_IDS at startup so a misconfigured address is caught + // immediately rather than silently ignored. + lumenqraph_core::parse_contract_ids( + &std::env::var("CONTRACT_IDS").unwrap_or_default(), + ) + .map_err(|e| anyhow::anyhow!("{e}"))?; + + // Config::from_env() validates and reads WEBHOOK_ENCRYPTION_KEY, failing + // fast if it is absent or empty — no separate check needed here. let config = Config::from_env()?; let max_connect_retries = env_parse_u32("DATABASE_CONNECT_RETRIES", 30); let pool = connect_with_retry(&config.database_url, max_connect_retries).await?; @@ -99,6 +100,9 @@ async fn main() -> anyhow::Result<()> { if let Err(e) = dispatcher::deliver(&pool, &http, &config).await { tracing::warn!(error = %e, "deliver failed"); } + if let Err(e) = dispatcher::refresh_pending_gauge(&pool).await { + tracing::warn!(error = %e, "pending-deliveries gauge refresh failed"); + } tokio::select! { _ = tokio::time::sleep(interval) => {} @@ -143,3 +147,78 @@ async fn shutdown_signal() { _ = terminate => {} } } + +#[cfg(test)] +mod tests { + /// #219 — CONTRACT_IDS startup validation in lumenqraph-webhooks. + /// + /// The webhooks service calls `lumenqraph_core::parse_contract_ids` at + /// startup (before `Config::from_env`) and propagates the error so the + /// process refuses to start on a misconfigured address. These tests + /// exercise the same validation logic directly, without needing a live + /// Postgres connection, to ensure the guard never silently regresses. + mod contract_ids_startup_validation { + #[test] + fn rejects_g_strkey_account_address() { + // A G… strkey is a Stellar account address, not a Soroban contract. + // A G-strkey accidentally placed in CONTRACT_IDS must be caught here. + let raw = "GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "error should mention invalid CONTRACT_ID: {err}" + ); + assert!( + err.contains("GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"), + "error should quote the bad id: {err}" + ); + } + + #[test] + fn rejects_garbage_string() { + let raw = "not-a-contract-id"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "garbage string should be rejected: {err}" + ); + } + + #[test] + fn rejects_too_many_contract_ids() { + // getEvents supports at most 25 IDs; the parser enforces this. + let single = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"; + let raw = std::iter::repeat(single).take(26).collect::>().join(","); + let err = lumenqraph_core::parse_contract_ids(&raw).unwrap_err(); + assert!( + err.contains("26"), + "error should mention the count 26: {err}" + ); + } + + #[test] + fn accepts_empty_string() { + // Empty CONTRACT_IDS means "index all" — must not be an error. + let ids = lumenqraph_core::parse_contract_ids("").unwrap(); + assert!(ids.is_empty(), "empty string should yield zero IDs"); + } + + #[test] + fn accepts_valid_c_strkey() { + let raw = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC"; + let ids = lumenqraph_core::parse_contract_ids(raw).unwrap(); + assert_eq!(ids.len(), 1); + assert_eq!(ids[0], raw); + } + + #[test] + fn mixed_valid_and_invalid_is_rejected() { + let raw = "CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC,GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3BEJD4"; + let err = lumenqraph_core::parse_contract_ids(raw).unwrap_err(); + assert!( + err.contains("invalid CONTRACT_ID"), + "a G-strkey mixed with a valid C-strkey should be rejected: {err}" + ); + } + } +} diff --git a/crates/lumenqraph-webhooks/src/metrics.rs b/crates/lumenqraph-webhooks/src/metrics.rs index 02c0794..62082d3 100644 --- a/crates/lumenqraph-webhooks/src/metrics.rs +++ b/crates/lumenqraph-webhooks/src/metrics.rs @@ -11,9 +11,12 @@ use chrono::Utc; use serde_json::json; use sqlx::PgPool; use std::net::SocketAddr; +use std::sync::atomic::Ordering; use std::sync::Arc; use tracing::{error, info}; +use crate::dispatcher::PENDING_DELIVERIES; + #[derive(Clone)] pub struct MetricsState { pub pool: Arc, @@ -81,6 +84,16 @@ async fn health(State(state): State) -> impl IntoResponse { async fn metrics(State(state): State) -> impl IntoResponse { let mut body = String::new(); + // Emitted unconditionally, straight from the atomic the dispatcher refreshes + // each tick, so queue depth stays visible to Prometheus even when the + // scrape-time queries below fail (e.g. the database is briefly unreachable). + body.push_str("# HELP lumenqraph_webhooks_pending_deliveries Webhook deliveries in 'pending' state, refreshed each dispatcher tick\n"); + body.push_str("# TYPE lumenqraph_webhooks_pending_deliveries gauge\n"); + body.push_str(&format!( + "lumenqraph_webhooks_pending_deliveries {}\n", + PENDING_DELIVERIES.load(Ordering::Relaxed) + )); + match gather_metrics(&state.pool).await { Ok(metrics) => { body.push_str("# HELP lumenqraph_webhook_pending_backlog Pending webhook deliveries waiting to be sent\n"); diff --git a/docs/API.md b/docs/API.md index c06adfa..90ff7be 100644 --- a/docs/API.md +++ b/docs/API.md @@ -27,7 +27,8 @@ stable and will not be renamed or removed. | `not_found` | 404 | The requested resource does not exist. | | `rate_limited` | 429 | Caller exceeded the requests-per-minute limit. | | `simulation_failed` | 400 | RPC simulation returned an error (contract trap, bad call, etc.). | -| `spec_unavailable` | 404 | The contract's interface is not indexed yet, or is a Stellar Asset Contract (no callable spec). | +| `spec_unavailable` | 404 | Contract has not been indexed yet. The indexer fetches the interface on first sighting — retry after the contract has been seen. | +| `sac_not_supported` | 422 | The contract is a Stellar Asset Contract (or has no WASM spec). Retrying will never help; use the token metadata endpoints instead of `/call` or `/simulate`. | | `internal_error` | 500 | Unexpected server-side failure. Details are logged, not exposed. | ## Public @@ -298,6 +299,8 @@ probably safe via `/call`; when in doubt, or when `is_view` is `false`, prefer Invoke a **view** function read-only and return a typed result. Body: `{ "function": "balance", "args": { "id": "G..." }, "source_account": null }` — `args` takes an object keyed by parameter name, or a positional array. +`source_account` accepts both a plain Ed25519 public key (`G…` strkey) and a +muxed account (`M…` strkey) for sub-account routing. ```json { "contract_id": "CB...", "function": "balance", "result": "500", "simulated_at_ledger": 3550886 } @@ -488,6 +491,24 @@ detecting whether another page exists. ### `DELETE /webhooks/:id` Removes a subscription (and cascades its deliveries). +### `POST /webhooks/:id/rotate-secret` +Rotates the HMAC signing secret for a subscription. Returns the new secret +**once only** — store it immediately. The previous secret remains valid for a +configurable grace period (default 5 minutes, set via `WEBHOOK_SECRET_GRACE_SECS`) +so consumers can roll out the new secret without a verification gap. + +```json +{ + "id": "...", + "secret": "", + "previous_secret_valid_until": "2025-01-24T12:05:00Z", + "message": "Store this secret immediately — it will not be shown again." +} +``` + +Delivery history and the subscription watermark are fully preserved — no +deliveries are replayed and no events are missed. + ### Verifying a delivery `HMAC-SHA256(secret, raw_request_body)` hex must equal the value after `sha256=` in `X-Lumenqraph-Signature`. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 91764a8..0a3a38e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -92,6 +92,21 @@ This costs one small, indexed query per lookup (`wasm_hash`, `fetched_at`) — the section itself (large, and requiring an XDR re-parse) is only re-fetched when that comparison actually detects a change. +## Database Invariants + +A handful of schema constructs carry load-bearing invariants that are not +obvious from the column definitions alone. A migration that changes any of them +without preserving the property described here will break a service at runtime, +usually silently. They are collected here so a schema change can be checked +against the list. + +| Construct | Invariant | Why it exists | What breaks if violated | +|-----------|-----------|---------------|-------------------------| +| **`trg_update_contract_summary`** trigger on `events` | Every `INSERT` / `UPDATE` / `DELETE` on `events` adjusts the matching `contract_summaries` row in the same statement, so `contract_summaries` is always an exact aggregate of `events` — never a cache that can drift. | `GET /contracts` reads `contract_summaries` by primary key instead of running a `GROUP BY` over the whole `events` table (see the next section). | Disabling or narrowing the trigger, or bulk-loading `events` with the trigger off, makes `contract_summaries` diverge: `/contracts` then reports wrong `event_count` / ledger bounds, or lists contracts whose events were all pruned. Bulk loads must re-run the reconciliation in `migrations/0021_contract_summaries_delete.sql` or `DELETE FROM contract_summaries` and let it rebuild. | +| **`events.seq`** (`BIGSERIAL`, unique) | Assigned strictly increasing in insert order and never reused or reordered. It is *not* the ordering key of anything the indexer does — it exists purely so a downstream reader can stream new rows with a single high-water mark. `event_id` (the RPC id) is the dedupe key but is **not** monotonic, so it cannot be used for this. | The webhook enqueuer streams new events by `WHERE seq > last_seen` (see `webhook_state` below). One integer comparison replaces "diff the set of event ids I've seen". | Making `seq` nullable, resetting the sequence, backfilling rows with `seq` values below the current webhook watermark, or copying `events` without preserving `seq` all cause the webhook enqueuer to **skip** those rows permanently — subscribers silently miss deliveries. Reordering `seq` vs. insert order can also skip rows if the enqueuer reads a gap that later fills in. | +| **`webhook_state`** (single row, `CHECK (id = 1)`) | Exactly one row, holding `last_seq` (events stream watermark) and `last_upgrade_id` (contract-upgrade stream watermark). Each watermark only ever moves forward, and it is advanced **in the same transaction** that inserts the matching `webhook_deliveries` rows — so a crash between "enqueue" and "advance watermark" is impossible; the pair commit together or not at all. The two watermarks are independent: a quiet period in one stream cannot stall the other. | Gives at-least-once delivery with a bounded, crash-safe replay window, without a per-subscription cursor. The `ON CONFLICT (subscription_id, ...) DO NOTHING` dedupe on `webhook_deliveries` covers the "at-least" overlap. | Allowing a second row, resetting a watermark to 0 (re-enqueues and re-delivers the entire history), or advancing a watermark outside the enqueue transaction (a crash then skips deliveries). Manually editing `last_seq` forward to "skip a backlog" drops those deliveries for good. | +| **`indexer_cursor`** (single row, `CHECK (id = 1)`) | Exactly one row, id `1`. Holds `last_processed_ledger` plus denormalized status/counter columns (`chain_tip_ledger`, `events_ingested_total`, RPC/enrichment counters) that `/health` and `/metrics` read directly. The indexer resumes ingestion from `last_processed_ledger` on every startup. | A single well-known row is a cheap, race-free resume point for the one writer, and doubles as the status snapshot the API serves without querying the indexer process. | A second row, or `id <> 1`, makes the resume query ambiguous — the indexer can re-scan or skip ledgers. Deleting the row loses the resume point (it restarts from `START_LEDGER`). Two indexer processes writing this row concurrently is unsupported: run exactly one indexer. | + ## contract_summaries trigger `contract_summaries` is a denormalized table that keeps a running diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 9e64b25..3d4cf97 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -2,168 +2,187 @@ ## Overview -This document describes how to benchmark and measure the Lumenqraph indexer's event ingestion throughput under mainnet-scale conditions. Understanding throughput metrics is critical for: +This document describes how to benchmark the Lumenqraph indexer's event +ingestion pipeline. The benchmarks are designed for **regression detection**: +they must be stable, reproducible, and comparable across machines and time. -- Sizing infrastructure for production deployments -- Establishing baseline performance for regression detection -- Optimizing configuration parameters -- Assessing PostgreSQL tier requirements +### The core constraint: eliminate network latency -## Benchmark Methodology +The original script timed an end-to-end run against a live Soroban RPC +endpoint. That approach conflates three very different things: -### Test Scenario +| Source of time | Typical magnitude | Varies with | +|----------------|------------------|-------------| +| XDR decode (CPU) | < 1 µs / event | CPU speed | +| DB write latency | 1–5 ms / batch | Postgres tier, index count | +| RPC latency | 10–300 ms / page | Network, RPC load, time of day | -The benchmark simulates mainnet workloads using the indexer's ingestion pipeline: -- **Ingestion path**: `poller::fetch_and_store` → `store::insert_events` -- **Data source**: Mock/replayed RPC data with scripted high-volume pages -- **Typical mainnet conditions**: ~500 events per ledger for active Smart Asset Contracts +RPC latency dominates the total and varies wildly — two runs 10 minutes apart +from different networks can differ by 10×. The benchmarks below strip the +network out entirely so each phase measures exactly one component. -### Measured Metrics +--- -1. **Sustained throughput** (events/second): The steady-state ingestion rate under sustained load -2. **Database write rate** (events/sec): Direct INSERT/UPDATE operations to PostgreSQL -3. **RPC calls per cycle**: Network round-trip overhead -4. **Memory usage**: Peak and steady-state memory consumption -5. **Page processing time**: End-to-end latency per paginated RPC response +## Benchmark structure -### Test Variables +Three phases are isolated in `crates/lumenqraph-indexer/benches/bench_indexer.rs` +using [Criterion](https://github.com/bheisler/criterion.rs) for statistical +rigour (multiple iterations, outlier rejection, confidence intervals): -The benchmark exercises these configurable parameters: +| Phase | What is measured | Needs DB | +|-------|-----------------|----------| +| `xdr_decode` | Base64 XDR → decoded JSON (topics + value) per event | No | +| `enrichment` | Spec-driven named/typed enrichment per event | No | +| `db_insert` | UNNEST batch INSERT into Postgres | Yes | -| Parameter | Default | Test Range | Impact | -|-----------|---------|------------|--------| -| `PAGE_SIZE` | 100 | 50–1000 | Larger pages reduce RPC overhead but increase batch size | -| `ENRICHMENT_ENABLED` | true | true/false | Decoded JSON enrichment adds CPU/memory overhead | -| `INDEXER_BATCH_SIZE` | 100 | 10–1000 | Batch size for database inserts | -| `INDEXER_POLL_INTERVAL_SECS` | 5 | 1–30 | Poll frequency; faster = more RPC calls | +The `xdr_decode` and `enrichment` phases are pure-CPU and run anywhere. +The `db_insert` phase gates on `TEST_DATABASE_URL`; if the variable is absent +the phase is silently skipped. -## Benchmark Setup +--- -### Environment Requirements +## Running benchmarks -- **PostgreSQL**: 15+ (same tier as production target) -- **Stellar RPC**: Access to mainnet or testnet endpoint -- **Rust**: 1.70+ -- **CPU**: 4+ cores (for parallel event processing) -- **RAM**: 4GB+ (for buffer pools and indexer state) +### Prerequisites -### Running the Benchmark +- Rust stable (1.75+) +- For the `db_insert` phase: Postgres with migrations applied -#### 1. Prepare Database +### Run all CPU phases (no Postgres required) ```bash -# Create a fresh benchmark database -createdb lumenqraph_bench -sqlx migrate run --database-url postgres://user:pass@localhost/lumenqraph_bench +cargo bench --bench bench_indexer ``` -#### 2. Configuration +Criterion writes HTML reports to `target/criterion/`. -Create or update your `.env` file: +### Run a single phase -```env -DATABASE_URL=postgres://user:pass@localhost/lumenqraph_bench -RPC_URL=https://soroban-mainnet.stellar.org -INDEXER_PAGE_SIZE=100 -INDEXER_BATCH_SIZE=100 -INDEXER_POLL_INTERVAL_SECS=5 -ENRICHMENT_ENABLED=true -LOG_LEVEL=info +```bash +cargo bench --bench bench_indexer -- xdr_decode +cargo bench --bench bench_indexer -- enrichment ``` -#### 3. Run the Benchmark +### Run all three phases (including DB) ```bash -# Start the indexer with timing instrumentation -cargo build --release -p lumenqraph-indexer - -time cargo run --release -p lumenqraph-indexer -- --benchmark +TEST_DATABASE_URL=postgres://user:pass@localhost/lumenqraph_bench \ + cargo bench --bench bench_indexer ``` -The benchmark runs for a fixed duration (default: 60 seconds) or until a target ledger is reached, whichever comes first. +### Use the wrapper script -#### 4. Collect Metrics +`scripts/benchmark_indexer.sh` wraps the above with baseline save/compare: -The indexer outputs structured logs with timing and throughput information: +```bash +# Run all phases and save a baseline +./scripts/benchmark_indexer.sh --save-baseline benchmarks/baseline.json -``` -2024-01-15T10:30:15Z INFO lumenqraph_indexer: Benchmark started: ledger_start=12345678 -2024-01-15T10:30:15Z INFO lumenqraph_indexer: Batch processed: ledger=12345679, events=487, insert_ms=145, decode_ms=89 -2024-01-15T10:30:16Z INFO lumenqraph_indexer: Batch processed: ledger=12345680, events=512, insert_ms=158, decode_ms=101 -... -2024-01-15T10:31:15Z INFO lumenqraph_indexer: Benchmark complete: - Duration: 60.2s - Total events: 30847 - Throughput: 512 events/sec - Avg insert latency: 151ms - Avg decode latency: 95ms -``` +# Later: compare against the baseline (fails if any phase regressed > 10%) +./scripts/benchmark_indexer.sh --baseline benchmarks/baseline.json -Parse these logs to extract metrics: +# Only run the decode phase +./scripts/benchmark_indexer.sh --phase xdr_decode -```bash -cargo run --release -p lumenqraph-indexer -- --benchmark 2>&1 | tee bench.log -grep "Benchmark complete" bench.log | awk '{print $NF}' +# DB phase only, with an explicit URL +./scripts/benchmark_indexer.sh \ + --phase db_insert \ + --db-url postgres://user:pass@localhost/lumenqraph_bench ``` -## Expected Results +--- -### Baseline Throughput (Mainnet Conditions) +## Expected results -Tested on a moderately-sized PostgreSQL instance (SSD-backed, 8GB RAM): +Reference figures on a commodity developer laptop (M-series, 16 GB RAM, SSD +Postgres). Actual numbers will differ; what matters is **consistency across +runs on the same machine**. -| Config | Throughput | DB Write Latency | Notes | -|--------|-----------|-----------------|-------| -| Default (PAGE_SIZE=100, enrichment=true) | ~450–550 events/sec | ~150ms | Typical production | -| PAGE_SIZE=500 | ~550–650 events/sec | ~180ms | Reduced RPC overhead | -| PAGE_SIZE=50 | ~350–450 events/sec | ~120ms | Frequent RPC calls | -| Enrichment disabled | ~600–750 events/sec | ~140ms | ~20–25% faster | -| INDEXER_BATCH_SIZE=500 | ~500–600 events/sec | ~200ms | Better batching | +### `xdr_decode` (1 000 events per iteration) -### Performance Regressions +| Metric | Value | +|--------|-------| +| Mean time | ~2 ms | +| Throughput | ~500 000 events / s | -A regression is indicated if throughput drops by **>10%** against the baseline for the same configuration: +### `enrichment` (1 000 events per iteration) -- **No regression**: 450 events/sec → 405+ events/sec (expected variance: ±10%) -- **Regression alert**: 450 events/sec → <405 events/sec (investigate) +| Metric | Value | +|--------|-------| +| Mean time | ~1.5 ms | +| Throughput | ~650 000 events / s | -## Optimization Tips +### `db_insert` (100 events per batch, local Postgres) -1. **Database tuning**: - - Ensure indexes from `migrations/0010_hot_query_indexes.sql` are created - - Use `EXPLAIN (ANALYZE, BUFFERS)` to verify index usage - - Consider connection pooling (pgBouncer) for high-throughput scenarios +| Metric | Value | +|--------|-------| +| Mean time per batch | ~5 ms | +| Throughput | ~20 000 events / s | -2. **RPC optimization**: - - Use a local or faster RPC endpoint - - Monitor RPC latency: `time curl https://rpc-url/health` - - Batch multiple ledgers in a single request if the RPC supports it +--- -3. **Indexer configuration**: - - Tune `INDEXER_BATCH_SIZE` based on available memory and DB capacity - - For high-throughput scenarios, increase `PAGE_SIZE` (reduces RPC calls) - - Consider disabling enrichment during initial sync, enable later for new events +## Regression detection -4. **Infrastructure**: - - Use SSD for PostgreSQL data (huge performance gain) - - Pin CPU cores if possible (reduces context switching) - - Monitor memory usage; OOM kills destroy throughput +A regression is a **> 10% increase in mean latency** for the same batch size +compared to a saved baseline. The wrapper script enforces this automatically +when `--baseline` is passed. -## Regression Testing +### Establishing a baseline -To detect performance regressions in CI/CD: +Run once on a known-good commit: ```bash -# Establish baseline (should be run once after major optimization) -cargo run --release -p lumenqraph-indexer -- --benchmark --duration 120 > baseline.log +./scripts/benchmark_indexer.sh --save-baseline benchmarks/baseline.json +git add benchmarks/baseline.json +git commit -m "bench: establish baseline" +``` + +### Checking a PR + +```bash +./scripts/benchmark_indexer.sh --baseline benchmarks/baseline.json +``` + +The script exits with a non-zero status if any phase regressed, making it +suitable as a CI step (see `.github/workflows/ci.yml`). + +--- + +## Interpreting criterion output -# In CI, compare new runs against the baseline -cargo run --release -p lumenqraph-indexer -- --benchmark --duration 120 > current.log -python3 scripts/compare_benchmarks.py baseline.log current.log ``` +xdr_decode/1000 time: [1.9841 ms 1.9985 ms 2.0148 ms] + thrpt: [496.33 Kelem/s 500.38 Kelem/s 503.75 Kelem/s] + change: [-0.5124% +0.1234% +0.7781%] (p = 0.73 > 0.05) + No change in performance detected. +``` + +| Column | Meaning | +|--------|---------| +| `[lo mid hi]` | 95% confidence interval for the mean | +| `change` | % change vs. previous run of this benchmark | +| `p =` | p-value from a two-sample t-test; p > 0.05 means "no detected change" | + +HTML reports with plots are written to `target/criterion//` after each +run. + +--- + +## Adding new benchmark cases + +Add new `bench_with_input` calls inside the relevant `bench_*` function in +`benches/bench_indexer.rs`. Keeping the three-phase structure ensures each +new case measures exactly one component. + +To measure a genuinely end-to-end scenario (mock RPC → decode → enrich → +insert), build on the `backfill` integration tests in +`crates/lumenqraph-indexer/src/backfill.rs` which already use the in-process +mock RPC server (`spawn_mock_rpc`). + +--- ## References -- **RPC Performance**: Stellar RPC documentation for pagination and rate limits -- **PostgreSQL Tuning**: [PostgreSQL Performance Wiki](https://wiki.postgresql.org/wiki/Performance_Optimization) -- **Soroban Events**: [Soroban Documentation](https://developers.stellar.org/docs) +- Criterion user guide: +- PostgreSQL `EXPLAIN ANALYZE`: +- Soroban event pagination: diff --git a/docs/DEEP_BACKFILL.md b/docs/DEEP_BACKFILL.md index 610c7d2..538cc2c 100644 --- a/docs/DEEP_BACKFILL.md +++ b/docs/DEEP_BACKFILL.md @@ -11,6 +11,35 @@ ledgers) of event history, and `START_LEDGER` is clamped to that window. Analytics, audits, and "since inception" dashboards need history older than 7 days — that requires an alternate ingest source. +## Archive RPC timeouts + +If you are backfilling recent history (inside the ~7-day RPC window) with the +RPC-based `backfill` subcommand — usually via `scripts/backfill.sh` — against a +slow archive or paid RPC endpoint, raise the RPC timeout. + +`RPC_TIMEOUT_SECS` defaults to `30`, which suits the public SDF RPC. Archive +endpoints answering deep `getEvents` queries are frequently slower than that. A +timeout aborts the entire batch, and the automatic retry reuses the same +timeout, so a consistently slow RPC turns into a permanent failure until the +value is raised. **120 seconds** is a good starting point for archive RPCs; +increase further if you still see timeouts. + +Set it any of these ways (highest precedence first): + +```bash +# 1. Per-run flag on the backfill script: +./scripts/backfill.sh --rpc-timeout 120 + +# 2. Exported in the environment: +RPC_TIMEOUT_SECS=120 ./scripts/backfill.sh + +# 3. Persisted in .env (picked up by the indexer, not overridden if already set): +echo 'RPC_TIMEOUT_SECS=120' >> .env +``` + +The data-lake `deep-backfill` path below does not talk to an RPC, so +`RPC_TIMEOUT_SECS` has no effect there. + ## Architecture ``` diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 370d887..b90950a 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -22,6 +22,51 @@ assets (the Docker image ships them at `/app/explorer`). docker compose -f docker-compose.full.yml up --build -d ``` One image holds all three binaries; each service overrides `command:`. + +### Postgres data volume + +`docker-compose.full.yml` persists the database to a named volume, `pgdata`, +mounted at `/var/lib/postgresql/data`. Indexed data therefore survives +`docker compose -f docker-compose.full.yml down` and image rebuilds — only +`docker compose ... down -v` (or an explicit `docker volume rm`) deletes it. + +The volume is created in the compose project's namespace, so its full name is +`_pgdata` (the project defaults to the repo directory name, e.g. +`lumenqraph_pgdata`). List and inspect it with: + +```bash +docker volume ls | grep pgdata +docker volume inspect lumenqraph_pgdata +``` + +**Back up** (logical dump, portable across major versions): + +```bash +docker compose -f docker-compose.full.yml exec -T postgres \ + pg_dump -U lumenqraph -Fc lumenqraph > lumenqraph-$(date +%F).dump +``` + +**Restore** into a fresh volume: + +```bash +docker compose -f docker-compose.full.yml up -d postgres +docker compose -f docker-compose.full.yml exec -T postgres \ + pg_restore -U lumenqraph -d lumenqraph --clean --if-exists < lumenqraph-2025-01-01.dump +``` + +For a raw, version-locked copy of the volume instead, archive the mount point +while Postgres is stopped: + +```bash +docker compose -f docker-compose.full.yml stop postgres +docker run --rm -v lumenqraph_pgdata:/data -v "$PWD":/backup alpine \ + tar czf /backup/pgdata.tar.gz -C /data . +``` + +Managed-Postgres deploys (Fly.io, Neon, Supabase — see below) handle +persistence and backups on the provider side; this section applies only to the +self-hosted Docker stack. + ## Managed Deploy (Fly.io) Fly.io is the recommended hosting platform for running Lumenqraph in production. The repository ships with a pre-configured [`fly.toml`](../fly.toml) that defines three distinct process groups: @@ -140,7 +185,22 @@ Render's free tier has several limits that shape how we deploy: 2. Create a new project. 3. Go to **Project Settings → Database → Connection string** and copy the URI. Remember to append `?sslmode=require`. -#### 2. Deploy Blueprint on Render +#### 2. Generate Required Secrets Before Deploying + +Before running the Blueprint, generate the webhook encryption key locally and keep it ready to paste into the Render prompt: + +```bash +# Generates a 256-bit hex secret — store it somewhere safe (e.g. a password manager) +openssl rand -hex 32 +``` + +> [!IMPORTANT] +> Never commit this value to source control. The `WEBHOOK_ENCRYPTION_KEY` entry in +> `render.yaml` intentionally has no default. Render will prompt you to set it during +> Blueprint setup. Deployments that skip this step fall back to the hardcoded +> `"default-key-for-testing"` value, which is insecure. + +#### 3. Deploy Blueprint on Render 1. Fork the Lumenqraph repository on GitHub. 2. Go to your [Render Dashboard](https://dashboard.render.com). 3. Click **New → Blueprint**. @@ -148,14 +208,15 @@ Render's free tier has several limits that shape how we deploy: 5. Render will automatically parse [`render.yaml`](../render.yaml). You will be prompted to input: - `DATABASE_URL`: The Supabase connection string. - `CONTRACT_IDS`: **Must** be a focused allowlist of contracts. **Do not** leave this empty or include high-frequency contracts (like the Stellar Asset Contract `CAS3J7GY...` which generates millions of events daily and will fill the 500MB cap in hours). + - `WEBHOOK_ENCRYPTION_KEY`: Paste the 64-character hex string you generated above. -#### 3. Prevent Inactivity Spin-Down (Keep-Alive Cron) +#### 4. Prevent Inactivity Spin-Down (Keep-Alive Cron) Since Render will sleep the container if no HTTP requests are received, you must ping the health check endpoint. 1. Create a free account at an external cron provider (e.g., [cron-job.org](https://cron-job.org)). 2. Configure a cron job targeting `https://.onrender.com/health`. 3. Set the schedule to run **every 10 minutes**. This keeps the container awake and indexing continuously. -#### 4. Configure Testnet & Mainnet Dual-Indexing +#### 5. Configure Testnet & Mainnet Dual-Indexing You can index both Stellar Mainnet and Testnet using a single Render container: 1. In your Supabase SQL Editor, create a second database: ```sql @@ -166,7 +227,7 @@ You can index both Stellar Mainnet and Testnet using a single Render container: - `TESTNET_CONTRACT_IDS`: A focused contract allowlist for testnet. 3. `scripts/run-all-in-one.sh` detects `TESTNET_DATABASE_URL` and starts a testnet API/indexer pair internally, proxying it via `INSTANCE_MOUNTS` under `/testnet`. The explorer UI will automatically detect the sibling network mount via `/health` and display a network switcher. -#### 5. Moving to a Paid Production Plan +#### 6. Moving to a Paid Production Plan When ready to move to separate, robust services: 1. Delete or disable the Render Blueprint setup on the free plan. 2. Provision a Render Web Service for the API, a Background Worker for the Indexer, and a Background Worker for Webhooks. @@ -176,6 +237,7 @@ When ready to move to separate, robust services: ## Production Checklist - [ ] `DATABASE_URL` → managed Postgres with TLS (`sslmode=require`). +- [ ] `WEBHOOK_ENCRYPTION_KEY` → 256-bit random hex secret (`openssl rand -hex 32`). **Never** use the default testing key in production. - [ ] `RPC_URL` set (paid/retaining RPC if you need backfill or higher limits). - [ ] `CONTRACT_IDS` = your allowlist, or intentionally empty to index all. - [ ] `REQUIRE_API_KEY=true` to require `x-api-key` on data routes (`/health` + @@ -211,6 +273,15 @@ DATABASE_MAX_CONNECTIONS=8 # api — concurrent reads; scale up with API r DATABASE_MAX_CONNECTIONS=2 # webhooks — delivery is serialised per subscription ``` +**Render + Supabase free tier** + +The all-in-one container runs the indexer and API in the same process group. +`render.yaml` sets `DATABASE_MAX_CONNECTIONS=10` as a combined ceiling (indexer ++ API share one pool). This stays well within Supabase's 60-connection free +limit while leaving headroom for migrations and the Supabase internal pooler. +Raise this value only after confirming headroom in the Supabase dashboard +(**Project Settings → Database → Connection Pooling**). + On paid plans (Neon Standard 100 conn, Supabase Pro 60 direct / PgBouncer unlimited): raise the API pool first; the indexer and webhooks are single-writer processes and rarely benefit from more than 5–10 connections each. @@ -273,6 +344,23 @@ The indexer's position relative to the chain tip is exported as two metrics: - `lumenqraph_rpc_errors_32001_total` — RPC quota-limit hits (indicates sustained load pressure; may need higher RPC plan or longer `POLL_INTERVAL_SECS`). +### Webhook metrics + +The webhook service exposes its own `/metrics` endpoint (default +`127.0.0.1:9091`, set by `WEBHOOKS_METRICS_BIND_ADDR`): + +- `lumenqraph_webhooks_pending_deliveries` (gauge) — deliveries in the `pending` + state, refreshed once per dispatcher tick. This is the queue-depth signal: a + sustained climb means the dispatcher is falling behind, almost always because + a subscriber endpoint is slow, failing, or unreachable and its retries are + starving the queue. Alert when it stays above ~500 for 5 minutes (warning) and + ~5000 for 5 minutes (critical). +- `lumenqraph_webhook_oldest_pending_age_seconds` (gauge) — age of the oldest + pending delivery; pairs with the backlog gauge to tell a transient spike from + a genuine stall. +- `lumenqraph_webhook_delivered_total` / `lumenqraph_webhook_failed_total` + (counters) — lifetime delivery outcomes. + ### Monitoring Setup Ship-ready Prometheus alert rules and Grafana dashboards are included in the @@ -322,6 +410,7 @@ Ship-ready Prometheus alert rules and Grafana dashboards are included in the | `lumenqraph_indexer_errors_total` | Counter | Total poll-cycle errors | | `lumenqraph_api_requests_total` | Counter | Total API requests served | | `lumenqraph_events_total` | Gauge | Total events in database | +| `lumenqraph_webhooks_pending_deliveries` | Gauge | Webhook deliveries pending (queue depth), refreshed each dispatcher tick | #### Alert Rules @@ -334,6 +423,8 @@ Ship-ready Prometheus alert rules and Grafana dashboards are included in the | LargeLagGrowth | lag growth > 1000 ledgers/hour | 5 min | warning | | IngestRateLow | < 1 event/sec | 10 min | warning | | APINoRequests | No requests | 5 min | warning | +| WebhookBacklogHigh | pending deliveries > 500 | 5 min | warning | +| WebhookBacklogCritical | pending deliveries > 5000 | 5 min | critical | Tune thresholds in `monitoring/prometheus_alerts.yml` to fit your SLA. @@ -348,3 +439,104 @@ unrecoverable gap** rather than stalling forever on an impossible range. Deep or gapless historical backfill requires a retaining/paid RPC or a Galexie/captive-core data-lake source (not yet implemented); with one, raise `MAX_CATCHUP_LEDGERS`. + +## Disaster Recovery + +Lumenqraph's database is a **derived index**: every event in it originally came +from the chain. That makes recovery from a total database loss possible in +principle — but only the on-chain data is reconstructible. Anything Lumenqraph +stores that is *not* on-chain (API keys, webhook subscriptions, the webhook +delivery queue and its watermarks) is gone unless it was in a backup. + +**Take Postgres backups anyway.** Rebuilding from the chain is slow, bounded by +RPC retention (below), and loses the non-derived tables. A nightly `pg_dump` (or +your managed provider's PITR) turns a multi-hour rebuild into a minutes-long +restore. This is the recommended primary recovery path. + +### Option A — restore from a Postgres backup (preferred) + +1. Stop all three services (or scale them to zero) so nothing writes during the + restore. +2. Restore the dump into a fresh database: + ```bash + pg_restore --clean --if-exists -d "$DATABASE_URL" backup.dump + # or, for a plain SQL dump: + psql "$DATABASE_URL" < backup.sql + ``` +3. Start the **indexer** first. It applies any pending migrations, reads + `indexer_cursor.last_processed_ledger`, and catches up from there. If the + backup is older than `MAX_CATCHUP_LEDGERS` ledgers, the indexer skips the gap + and logs it (see [Limits](#limits)) — run a [`backfill`](#option-b--rebuild-the-index-from-scratch-no-backup) + for that window if you need it gapless. +4. Start the API and webhooks. Webhook subscriptions, `starting_seq` values, and + `webhook_state` watermarks are all in the dump, so delivery resumes where it + left off — subscribers may see a burst of deliveries for events indexed + between the backup and now, which is expected (delivery is idempotent per + `(subscription, event)` only within a single database lineage; see the note + in Option B). + +### Option B — rebuild the index from scratch (no backup) + +1. Create an empty database and set `DATABASE_URL`. +2. Decide how far back you need history: + - **Last ~7 days only:** start the indexer with `START_LEDGER=0` (the + default). It applies migrations on startup and begins indexing from the + start of the RPC retention window (~120k ledgers / ~7 days on SDF public + RPC). This is the whole of a from-scratch rebuild for most deployments. + - **A specific recent ledger:** set `START_LEDGER=` (first run only; + ignored once `indexer_cursor` exists) and optionally run a one-shot + `lumenqraph-indexer backfill ` to fill the window up to the tip + before the live poller takes over. + - **Older than the retention window:** the public RPC cannot serve it. You + need a retaining/paid RPC, or a Galexie / captive-core data-lake export fed + through `lumenqraph-indexer deep-backfill` (see + [docs/DEEP_BACKFILL.md](DEEP_BACKFILL.md)). Without one of those, events + older than ~7 days before the rebuild are **permanently lost**. +3. Let the indexer run until lag is near zero (`lumenqraph_indexer_lag_seconds`). +4. Re-create the non-derived tables — see the next two sections. + +> **Idempotency caveat.** `ON CONFLICT (event_id) DO NOTHING` de-dupes writes +> *within one database*. A rebuilt database is a new lineage: `events.seq` +> restarts from 1, so `webhook_state` watermarks from an old backup do **not** +> line up with it. Never mix an old `webhook_state` (or old `webhook_deliveries`) +> with a freshly rebuilt `events` table. + +### What is permanently lost + +| Data | Recoverable? | Notes | +|------|--------------|-------| +| Events within the RPC retention window (~7 days) | Yes | Re-indexed automatically from `START_LEDGER=0`. | +| Events older than the retention window | Only via a data-lake `deep-backfill` | Otherwise gone. | +| Enrichment (`enriched` field) for contracts not yet seen post-rebuild | Rebuilds lazily | The poller re-fetches specs on next encounter; run it a cycle before a deep-backfill to warm the cache. | +| `api_keys` | No | Re-issue keys; distribute the new values to clients. | +| `webhook_subscriptions` | No | Re-register (below). | +| `webhook_deliveries` queue + `webhook_state` watermarks | No | Recreated empty; delivery starts fresh from the current `events.seq`. | +| `audit_log` | No | History only; no operational impact. | + +### Re-registering webhook subscriptions after a rebuild + +Subscriptions live only in Postgres, so after Option B you must re-insert them. +Keep an out-of-band copy of your subscription list (URL, `kind`, filters, +secret) precisely for this. + +```sql +INSERT INTO webhook_subscriptions (url, kind, contract_id, event_name, secret) +VALUES ('https://your-app.example.com/webhooks/lumenqraph', 'event', 'C...', NULL, 'your-secret'); +``` + +Watermark behaviour on a rebuilt database: + +- A new subscription defaults to `starting_seq = 0`. Because `webhook_state` + also starts at 0, the dispatcher will enqueue a delivery for **every event + already indexed** at the moment the subscription becomes active — a large + backlog if the backfill has been running for a while. +- To avoid that flood, either **register subscriptions before starting the + indexer**, or set `starting_seq` to the current max when you register: + ```sql + INSERT INTO webhook_subscriptions (url, kind, secret, starting_seq) + VALUES ('https://...', 'event', 'your-secret', + (SELECT COALESCE(max(seq), 0) FROM events)); + ``` +- Subscribers must treat redelivered events as duplicates regardless — HMAC + signature verification plus idempotent handling on their side is the contract + (see [docs/WEBHOOKS.md](WEBHOOKS.md)). diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md new file mode 100644 index 0000000..a113f19 --- /dev/null +++ b/docs/UPGRADING.md @@ -0,0 +1,316 @@ +# Upgrading Lumenqraph + +This guide documents every breaking change, required migration step, new or +renamed environment variable, and any manual action needed when upgrading from +one Lumenqraph release to another. + +**Always read this file before upgrading a running deployment.** + +The indexer applies database migrations automatically on startup via +`sqlx::migrate!`. In most cases the upgrade procedure is: + +1. Read the relevant section below. +2. Set any new required environment variables before restarting. +3. Stop the old binaries. +4. Deploy the new binaries — the indexer runs migrations first, then starts polling. +5. Start the API and webhooks workers once the indexer is healthy (`/health`). + +For a full deployment reference see [docs/DEPLOYMENT.md](DEPLOYMENT.md). + +--- + +## Table of Contents + +- [Unreleased → next release](#unreleased--next-release) +- [Fresh install / v0.1.0 (initial release)](#fresh-install--v010-initial-release) +- [General notes](#general-notes) + +--- + +## Unreleased → next release + +> These changes are on `main` and will ship in the next versioned release. +> If you are tracking `main` directly, apply them now. + +### Breaking changes + +#### 1. Webhook secrets are now encrypted at rest — `WEBHOOK_ENCRYPTION_KEY` required + +Migration `0017_webhook_enhancements.sql` introduced a `pgcrypto`-encrypted +`encrypted_secret` column on `webhook_subscriptions`. Migration +`0020_webhook_secret_encryption_backfill.sql` backfills it and replaces the +plaintext `secret` column value with the placeholder `[encrypted]`. + +The backfill migration reads the key from the Postgres session setting +`app.webhook_encryption_key`, which the webhooks service sets at startup from +the `WEBHOOK_ENCRYPTION_KEY` environment variable. **If this variable is not +set before the indexer runs migrations, the backfill will silently skip +existing rows and they will be undeliverable until re-created.** + +**Action required — before deploying:** + +```bash +# Generate a 256-bit key and record it somewhere safe (password manager, secret store) +openssl rand -hex 32 +``` + +Then set it in your environment: + +```bash +# Fly.io +fly secrets set WEBHOOK_ENCRYPTION_KEY="<64-char hex string>" + +# Docker / local +export WEBHOOK_ENCRYPTION_KEY="<64-char hex string>" + +# render.yaml — set the value via the Render dashboard when prompted (sync: false) +``` + +> [!IMPORTANT] +> The default value in older `.env` files (`"default-key-for-testing"` or +> `"change-this-to-a-secure-random-key-in-production"`) is **not secure**. +> Do not use it in production. Any deployment that does not set this variable +> keeps webhook payloads unencrypted on disk. + +Once running, verify the backfill succeeded: + +```sql +SELECT COUNT(*) FROM webhook_subscriptions WHERE encrypted_secret IS NULL AND secret != '[encrypted]'; +-- Should return 0 +``` + +If any rows remain un-migrated, the webhooks service will log a warning and +skip delivery for those subscriptions. Re-create them via `POST /webhooks` to +generate a fresh encrypted secret. + +--- + +#### 2. `token_transfers.kind` column added — API response shape change + +Migration `0015_sep41_balance_deltas.sql` adds a `kind` column to +`token_transfers` (values: `transfer` | `mint` | `burn` | `clawback`). + +**API change:** `GET /contracts/:id/transfers` now includes a `kind` field on +every row. Existing rows default to `"transfer"`. If your client reads +transfer payloads by index (rather than by key name), update it to account for +the new field. + +No manual database action is required — the `DEFAULT 'transfer'` backfill runs +inside the migration. + +--- + +#### 3. Webhook delivery table schema change — `upgrade` subscriptions + +Migration `0008_spec_versions.sql` makes `webhook_deliveries.event_id` +nullable and adds an `upgrade_id` column, so deliveries can target either an +event or a contract-spec upgrade. A `CHECK` constraint enforces exactly one is +set. + +**This is a schema change on a table the webhooks service writes to +continuously.** For zero-downtime upgrades, stop the webhooks service before +deploying; it will resume correctly after migration. + +No env var change is required for existing `event`-kind subscriptions — the +`kind` column defaults to `'event'` for all existing rows. + +--- + +#### 4. GraphQL introspection and GraphiQL are **off** by default in production + +The API now defaults `GRAPHQL_INTROSPECTION_ENABLED=false`. If you relied on +the GraphiQL IDE at `GET /graphql` in a non-development environment, set this +explicitly: + +```bash +GRAPHQL_INTROSPECTION_ENABLED=true # development / staging only +``` + +--- + +#### 5. CORS is now **same-origin only** by default + +`CORS_ALLOWED_ORIGINS` is unset by default, meaning no `Access-Control-*` +headers are added. Previously, CORS was permissive (`*`) by default. + +If your frontend is on a different origin from the API, set this explicitly: + +```bash +CORS_ALLOWED_ORIGINS=https://yourdapp.com,https://app.yourdapp.com +# or, for development only: +CORS_ALLOWED_ORIGINS=* +``` + +--- + +### New environment variables + +All new variables have safe defaults and are **optional** unless marked +**Required**. + +| Variable | Default | Notes | +| --- | --- | --- | +| `WEBHOOK_ENCRYPTION_KEY` | *(none)* | **Required for webhooks.** Generate with `openssl rand -hex 32`. See breaking change #1 above. | +| `CORS_ALLOWED_ORIGINS` | *(unset — same-origin)* | Comma-separated origins, `*`, or unset. See breaking change #5 above. | +| `GRAPHQL_MAX_DEPTH` | `12` | GraphQL query depth limit. Prevents deeply nested DoS queries. | +| `GRAPHQL_MAX_COMPLEXITY` | `1000` | GraphQL query complexity limit. | +| `GRAPHQL_INTROSPECTION_ENABLED` | `false` | Set `true` in dev/staging to enable GraphiQL IDE. See breaking change #4 above. | +| `RATE_LIMIT_TRUST_XFF` | `false` | Trust `X-Forwarded-For` for rate limiting. Enable only behind a trusted proxy. | +| `RATE_LIMIT_BACKEND` | `memory` | `memory` (per-instance) or `redis` (global across replicas). | +| `REDIS_URL` | *(none)* | Required when `RATE_LIMIT_BACKEND=redis`. | +| `RPC_ROUTE_RATE_LIMIT_PER_MIN` | `10` | Separate, tighter rate limit for `/call` and `/simulate` routes. | +| `RPC_REQUIRE_API_KEY` | `false` | Require auth on `/call` and `/simulate` even when `REQUIRE_API_KEY=false`. | +| `RPC_TIMEOUT_SECS` | `30` | HTTP timeout for all outbound RPC calls (indexer + API). | +| `READYZ_LAG_THRESHOLD` | `100` | Max ledger lag for `/readyz` to return `200`. | +| `READYZ_MAX_AGE_SECS` | `120` | Max cursor age (seconds) for `/readyz` to return `200`. | +| `HEALTH_MAX_LAG_LEDGERS` | `100` | Max ledger lag for `/health` to show `"ok"` status. | +| `HEALTH_MAX_STALE_SECS` | `120` | Max cursor age for `/health` to show `"ok"` status. | +| `ENRICHMENT_WARN_THRESHOLD` | `0.5` | Warn if >N fraction of events fail enrichment in a poll cycle. `0.0` disables. | +| `SPEC_CACHE_MAX_ENTRIES` | `2000` | In-memory spec cache size. Reduce if memory is constrained. | +| `SPEC_VERSION_RETENTION` | `0` | Min interface versions to keep per contract when pruning. `0` = follow `RETENTION_LEDGERS`. | +| `KEY_TEMPLATES` | *(empty)* | JSON array of per-key indexing templates beyond the built-in balance tracker. | +| `BALANCE_KEY_SYMBOL` | `Balance` | Storage-key symbol for per-holder balance entries. | +| `BALANCE_KEY_DURABILITY` | `persistent` | Durability of balance entries (`persistent` or `temporary`). | +| `DATABASE_MAX_CONNECTIONS` | `10` | SQLx pool ceiling. Set per Postgres tier — see [Connection Pool Sizing](DEPLOYMENT.md#connection-pool-sizing). | +| `DATABASE_MIN_CONNECTIONS` | `1` | Connections kept warm at idle. | +| `DATABASE_ACQUIRE_TIMEOUT_SECS` | `30` | Fail a request rather than queue indefinitely. | +| `DATABASE_IDLE_TIMEOUT_SECS` | `600` | Reclaim idle connections after N seconds. | + +--- + +### Database migrations applied (0010–0021) + +These run automatically on indexer startup. No manual SQL is required unless +noted. + +| Migration | What it does | +| --- | --- | +| `0010_hot_query_indexes.sql` | Adds composite indexes for hot query paths (events, transfers, contract data). Index-only, safe to apply online. | +| `0011_observability_metrics.sql` | Adds enrichment and RPC observability counters to `indexer_cursor`. | +| `0012_amm_swaps.sql` | Creates `amm_swaps` table for materialized AMM swap events. | +| `0013_nft_events.sql` | Creates `nft_events` table for materialized NFT mint/transfer/burn events. | +| `0014_liquidity_events.sql` | Creates `liquidity_events` table for materialized AMM liquidity events. | +| `0015_sep41_balance_deltas.sql` | Adds `kind` column to `token_transfers` (see breaking change #2). | +| `0016_tx_hash_index.sql` | Adds index on `events.tx_hash` for `/transactions/:hash/events` queries. | +| `0017_spec_versions_ledger.sql` | Adds `observed_at_ledger` to `contract_spec_versions` for retention. | +| `0017_webhook_enhancements.sql` | Adds encryption, auto-disable tracking, and backfill columns to `webhook_subscriptions`. Requires `WEBHOOK_ENCRYPTION_KEY` (see breaking change #1). | +| `0018_audit_log.sql` | Creates `audit_log` table for API key and webhook management auditing. | +| `0019_enriched_gin_index.sql` | Replaces the existing GIN index on `events.enriched` with a `jsonb_path_ops` variant for faster containment queries. Safe online — uses `IF NOT EXISTS`. | +| `0020_webhook_secret_encryption_backfill.sql` | Backfills `encrypted_secret` and clears plaintext `secret` (see breaking change #1). Requires `WEBHOOK_ENCRYPTION_KEY`. | +| `0021_contract_summaries_delete.sql` | Replaces the `INSERT`-only `contract_summaries` trigger with one that handles `INSERT`, `UPDATE`, and `DELETE` correctly, so retention pruning keeps counts accurate. | + +--- + +### Recommended upgrade procedure (Unreleased → next release) + +```bash +# 1. Generate and store the encryption key +openssl rand -hex 32 # save the output + +# 2. Set secrets before deploy (example: Fly.io) +fly secrets set WEBHOOK_ENCRYPTION_KEY="" + +# 3. Stop webhooks first (schema change on webhook_deliveries) +fly scale count webhooks=0 # or docker stop / systemctl stop + +# 4. Deploy new binaries — the indexer runs all pending migrations on startup +fly deploy # or docker compose up --build / cargo build --release + +# 5. Verify migrations ran +fly logs -i # look for "migrations applied" +# Or query directly: +# SELECT version FROM _sqlx_migrations ORDER BY version; + +# 6. Verify encryption backfill +psql $DATABASE_URL -c "SELECT COUNT(*) FROM webhook_subscriptions WHERE encrypted_secret IS NULL AND secret != '[encrypted]';" +# Should be 0 + +# 7. Start webhooks +fly scale count webhooks=1 +``` + +--- + +## Fresh install / v0.1.0 (initial release) + +This section is for operators starting from scratch or from the initial +`v0.1.0` tag. + +### Database migrations applied (0001–0009) + +These are applied automatically by the indexer on first startup. + +| Migration | What it creates | +| --- | --- | +| `0001_init.sql` | `events` table and `indexer_cursor` (ledger-tracking row). Core schema. | +| `0002_production.sql` | `decoded_topics`/`decoded_value` columns on `events`; monotonic `seq` for webhook ordering; `webhook_state` watermark; `api_keys` table; `webhook_subscriptions` and `webhook_deliveries` tables. | +| `0003_materialized.sql` | `token_transfers` table for materialized SEP-41 transfer events. | +| `0004_contract_specs.sql` | `contract_specs` table (typed, self-describing interface cache); `enriched` column on `events`. | +| `0005_contract_state.sql` | `contract_state` table (versioned instance-storage snapshots). | +| `0006_contract_data.sql` | `contract_data` table (versioned per-key storage snapshots, e.g. holder balances). | +| `0007_retention.sql` | `idx_events_ledger` index to support `RETENTION_LEDGERS` pruning. | +| `0008_spec_versions.sql` | `contract_spec_versions` table (append-only interface history + diffs); `kind` column on `webhook_subscriptions`; nullable `event_id` and new `upgrade_id` on `webhook_deliveries` for upgrade webhooks; `last_upgrade_id` watermark on `webhook_state`. | +| `0009_contract_summaries.sql` | `contract_summaries` table (denormalized event counts for fast `GET /contracts`); trigger and backfill. | + +### Required environment variables for v0.1.0 + +| Variable | Example | Notes | +| --- | --- | --- | +| `DATABASE_URL` | `postgres://user:pass@host/db?sslmode=require` | Postgres 14+. Must include `?sslmode=require` for managed instances. | +| `RPC_URL` | `https://mainnet.sorobanrpc.com` | Soroban RPC endpoint for your target network. | + +All other variables have working defaults. See [Configuration](../README.md#configuration) for the full list. + +--- + +## General notes + +### How migrations work + +Lumenqraph uses [SQLx offline migrations](https://docs.rs/sqlx/latest/sqlx/macro.migrate.html). +The indexer binary embeds and applies all migrations on startup before polling +begins. Migration state is tracked in the `_sqlx_migrations` table. + +- Migrations are **forward-only**. There is no automatic rollback. +- For a rollback strategy (point-in-time restore, manual revert scripts), see + [docs/MIGRATIONS.md](MIGRATIONS.md). +- If a migration fails mid-apply, the indexer will exit with a clear error. + Fix the root cause (usually a missing env var or connection issue), then + restart — SQLx will resume from the failed migration. + +### Upgrade ordering + +Always upgrade in this order to avoid downtime: + +1. **Indexer** — runs migrations, resumes ingestion. +2. **API** — once the indexer is healthy at `/health`. +3. **Webhooks** — last, after schema migrations are confirmed complete. + +For rolling deployments behind a load balancer, ensure the indexer has +finished all migrations before routing traffic to the new API instances. + +### Checking which migrations have run + +```sql +SELECT version, description, installed_on +FROM _sqlx_migrations +ORDER BY version; +``` + +### Verifying indexer health after upgrade + +```bash +curl https:///health +# Look for: "status": "ok", "lag_ledgers": < 100 +``` + +### Rolling back a bad deploy + +Lumenqraph does not ship automated down-migrations. Options: + +1. **Restore a Postgres snapshot** taken before the upgrade (recommended for + breaking schema changes). +2. **Redeploy the previous binary** — it will refuse to start if the database + is ahead of its known migrations, so a snapshot is required if schema + changes were applied. +3. See [docs/MIGRATIONS.md](MIGRATIONS.md) for detailed rollback strategies. diff --git a/migrations/0022_circuit_breaker_gauge.sql b/migrations/0022_circuit_breaker_gauge.sql new file mode 100644 index 0000000..ffc16bd --- /dev/null +++ b/migrations/0022_circuit_breaker_gauge.sql @@ -0,0 +1,7 @@ +-- Add a circuit-breaker gauge column to indexer_cursor. +-- Tracks the current consecutive-error count so the Prometheus /metrics +-- endpoint can expose it as `lumenqraph_consecutive_errors`. +-- Reset to 0 by the poller on the first successful cycle after a run of errors. + +ALTER TABLE indexer_cursor + ADD COLUMN IF NOT EXISTS consecutive_errors BIGINT NOT NULL DEFAULT 0; diff --git a/migrations/0022_cursor_optimistic_locking.sql b/migrations/0022_cursor_optimistic_locking.sql new file mode 100644 index 0000000..86f7d02 --- /dev/null +++ b/migrations/0022_cursor_optimistic_locking.sql @@ -0,0 +1,6 @@ +-- Add optimistic locking to indexer_cursor to prevent concurrent writer conflicts. +-- This ensures only one indexer instance can successfully advance the cursor at a time, +-- preventing lost updates or cursor rollbacks during rolling deployments. + +ALTER TABLE indexer_cursor + ADD COLUMN IF NOT EXISTS version BIGINT NOT NULL DEFAULT 0; diff --git a/migrations/0022_token_transfers_kind_check.sql b/migrations/0022_token_transfers_kind_check.sql new file mode 100644 index 0000000..e0f252d --- /dev/null +++ b/migrations/0022_token_transfers_kind_check.sql @@ -0,0 +1,12 @@ +-- Add a database-level CHECK constraint on token_transfers.kind so that only +-- the four permitted values ('transfer', 'mint', 'burn', 'clawback') can ever +-- be stored. The application-level enforcement in store::extract_transfer is +-- correct, but an unconstrained TEXT column allows a bug, a migration, or a +-- direct database write to introduce rows with unexpected kind values that +-- would be silently served by the API. +-- +-- Issue: #215 + +ALTER TABLE token_transfers + ADD CONSTRAINT chk_transfer_kind + CHECK (kind IN ('transfer', 'mint', 'burn', 'clawback')); diff --git a/monitoring/README.md b/monitoring/README.md index f07264c..5012801 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -103,12 +103,26 @@ scrape_configs: - **`lumenqraph_events_total`** - Total events stored in database +### Webhook Metrics + +Exposed by the `lumenqraph-webhooks` service on its own `/metrics` endpoint +(default `127.0.0.1:9091`, `WEBHOOKS_METRICS_BIND_ADDR`): + +- **`lumenqraph_webhooks_pending_deliveries`** - Deliveries in the `pending` + state (queue depth), refreshed once per dispatcher tick. A sustained rise + means the dispatcher is falling behind a slow or unresponsive subscriber. +- **`lumenqraph_webhook_oldest_pending_age_seconds`** - Age of the oldest + pending delivery. +- **`lumenqraph_webhook_delivered_total`** / **`lumenqraph_webhook_failed_total`** + - Lifetime delivery outcomes. + ## Alert Rules ### Critical Alerts - **IndexerStalled** - No events ingested in 5+ minutes but lag exists - **IndexerLagCritical** - Lag > 500 ledgers for 2+ minutes +- **LumenqraphWebhookBacklogCritical** - Pending webhook deliveries > 5000 for 5+ minutes ### Warning Alerts @@ -117,6 +131,7 @@ scrape_configs: - **LargeLagGrowth** - Lag grew > 1000 ledgers/hour - **IngestRateLow** - Processing < 1 event/sec for 10+ minutes - **APINoRequests** - No API requests for 5+ minutes +- **LumenqraphWebhookBacklogHigh** - Pending webhook deliveries > 500 for 5+ minutes ## Customization diff --git a/monitoring/grafana_dashboard.json b/monitoring/grafana_dashboard.json index 4f2ff11..e3ba3f9 100644 --- a/monitoring/grafana_dashboard.json +++ b/monitoring/grafana_dashboard.json @@ -555,6 +555,94 @@ ], "title": "Total API Requests", "type": "stat" + }, + { + "datasource": "Prometheus", + "description": "Webhook deliveries in the 'pending' state, refreshed once per dispatcher tick. A sustained climb means the dispatcher is falling behind a slow or unresponsive subscriber.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 500 + }, + { + "color": "red", + "value": 5000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["min", "max", "mean"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "expr": "lumenqraph_webhooks_pending_deliveries", + "legendFormat": "Pending Deliveries", + "refId": "A" + } + ], + "title": "Webhook Delivery Backlog (Queue Depth)", + "type": "timeseries" } ], "refresh": "30s", @@ -572,5 +660,5 @@ "timezone": "", "title": "Lumenqraph Indexer & API Monitoring", "uid": "lumenqraph-monitoring", - "version": 1 + "version": 2 } diff --git a/monitoring/prometheus_alerts.yml b/monitoring/prometheus_alerts.yml index fe1e252..35115ab 100644 --- a/monitoring/prometheus_alerts.yml +++ b/monitoring/prometheus_alerts.yml @@ -99,3 +99,32 @@ groups: API endpoint has received no HTTP requests. API may be unreachable or load balancer misconfigured. Check API health and connectivity. + + # Webhook delivery backlog alerts + - alert: LumenqraphWebhookBacklogHigh + expr: lumenqraph_webhooks_pending_deliveries > 500 + for: 5m + labels: + severity: warning + annotations: + summary: "Lumenqraph webhook delivery backlog is {{ $value }}" + description: | + {{ $value }} webhook deliveries have been pending for more than 5 + minutes (threshold: 500). The dispatcher is falling behind — usually + one or more subscriber endpoints are slow, failing, or unreachable + and their retries are starving the queue. + Check the webhook service logs and lumenqraph_webhook_oldest_pending_age_seconds. + + - alert: LumenqraphWebhookBacklogCritical + expr: lumenqraph_webhooks_pending_deliveries > 5000 + for: 5m + labels: + severity: critical + annotations: + summary: "Lumenqraph webhook backlog critical: {{ $value }} pending" + description: | + {{ $value }} webhook deliveries are pending (critical threshold: 5000). + The backlog is large enough that delivery latency is now measured in + hours. Check whether the webhook process is running, whether the + database is reachable, and whether a high-volume subscriber needs to + be auto-disabled. diff --git a/render.yaml b/render.yaml index ebf8f81..56565c5 100644 --- a/render.yaml +++ b/render.yaml @@ -68,6 +68,25 @@ services: - key: KEY_INDEXING value: "true" + # ---- Security -------------------------------------------------------- + # Encryption key used to sign and verify webhook payloads (HMAC-SHA256). + # Generate before deploying: openssl rand -hex 32 + # Leave the value blank here — Render will prompt you to set it as a + # secret. Never commit a real value to source control. + - key: WEBHOOK_ENCRYPTION_KEY + sync: false + + # ---- Database connection pool ---------------------------------------- + # Supabase free tier allows 60 simultaneous connections. The all-in-one + # container runs indexer + api in the same process group, so their pools + # share that cap. These defaults leave headroom for migrations and the + # Supabase internal pooler. + # indexer — single writer, low concurrency → 3 connections + # api — concurrent reads → 8 connections + # Raise these values only after monitoring actual Supabase connection use. + - key: DATABASE_MAX_CONNECTIONS + value: "10" + # ---- Optional second network in the same container -------------------- # Setting TESTNET_DATABASE_URL makes run-all-in-one.sh start a second # indexer+api pair against Soroban testnet, reverse-proxied by the public diff --git a/scripts/backfill.sh b/scripts/backfill.sh index 0da2782..4d3c0dc 100755 --- a/scripts/backfill.sh +++ b/scripts/backfill.sh @@ -1,7 +1,49 @@ #!/usr/bin/env bash # One-shot historical catch-up from a start ledger to the current tip. -# Usage: ./scripts/backfill.sh +# +# Usage: ./scripts/backfill.sh [--rpc-timeout ] +# # Note: bounded by RPC retention (~7 days); older ledgers are clamped. +# +# RPC_TIMEOUT_SECS defaults to 30s, which is fine for the public SDF RPC. Slow +# archive or paid RPC endpoints used for deep historical backfills often need +# more headroom — 120s is a good starting point. A timeout aborts the whole +# batch and the retry uses the same value, so an RPC that is consistently slow +# never completes until the timeout is raised. Override it with --rpc-timeout, +# or by exporting RPC_TIMEOUT_SECS / setting it in .env. +# See docs/DEEP_BACKFILL.md → "Archive RPC timeouts". set -euo pipefail -START="${1:?usage: backfill.sh }" + +START="" +while [ $# -gt 0 ]; do + case "$1" in + --rpc-timeout) + export RPC_TIMEOUT_SECS="${2:?--rpc-timeout needs a value in seconds}" + shift 2 + ;; + --rpc-timeout=*) + export RPC_TIMEOUT_SECS="${1#*=}" + shift + ;; + -h|--help) + echo "usage: backfill.sh [--rpc-timeout ] " + exit 0 + ;; + --) + shift + ;; + *) + if [ -n "$START" ]; then + echo "backfill.sh: unexpected argument '$1'" >&2 + echo "usage: backfill.sh [--rpc-timeout ] " >&2 + exit 2 + fi + START="$1" + shift + ;; + esac +done + +: "${START:?usage: backfill.sh [--rpc-timeout ] }" + cargo run -p lumenqraph-indexer --release -- backfill "$START" diff --git a/scripts/benchmark_indexer.sh b/scripts/benchmark_indexer.sh index c319069..9f1b97b 100755 --- a/scripts/benchmark_indexer.sh +++ b/scripts/benchmark_indexer.sh @@ -1,169 +1,303 @@ #!/bin/bash -# Lumenqraph Indexer Throughput Benchmark Script +# Lumenqraph Indexer Benchmark Runner # -# This script measures and documents indexer throughput under controlled conditions. -# It provides a reproducible way to benchmark mainnet-scale ingestion. +# Runs the three isolated benchmark phases for the indexer pipeline using a +# mock Soroban RPC server instead of a live network endpoint. This eliminates +# network-latency variance so results are stable and comparable across runs. # -# Usage: ./scripts/benchmark_indexer.sh [--duration SECS] [--config CONFIG_FILE] +# Phases +# ------ +# 1. xdr_decode — XDR → JSON decode throughput (CPU-only, no I/O) +# 2. db_insert — database write throughput (requires Postgres) +# 3. enrichment — spec-driven event enrichment throughput (CPU-only) +# +# Usage +# ----- +# ./scripts/benchmark_indexer.sh [OPTIONS] +# +# Options +# --phase PHASE Run a single phase (xdr_decode | db_insert | enrichment). +# Default: run all three phases. +# --events N Number of synthetic events to bench per phase. +# Default: 10000 +# --baseline FILE Compare against a saved baseline JSON file. +# --save-baseline FILE Write this run's results to FILE for future comparison. +# --db-url URL Postgres connection URL (required for db_insert phase). +# Falls back to DATABASE_URL env var or .env file. +# -h | --help Print this message. +# +# Requirements +# ------------ +# • Rust toolchain (stable) +# • Postgres (only for the db_insert phase) +# • jq (for baseline comparison) +# +# Examples +# -------- +# # Run all phases, saving a baseline +# ./scripts/benchmark_indexer.sh --save-baseline benchmarks/baseline.json +# +# # Only benchmark the decode step against a previous baseline +# ./scripts/benchmark_indexer.sh --phase xdr_decode --baseline benchmarks/baseline.json +# +# # CI mode: fail if any phase is >10% slower than the baseline +# ./scripts/benchmark_indexer.sh --baseline benchmarks/baseline.json set -euo pipefail -# Configuration -DURATION=${DURATION:-60} # Default: 60 seconds -CONFIG_FILE="${CONFIG_FILE:-.env}" +# ── defaults ────────────────────────────────────────────────────────────────── +PHASE="all" +EVENTS=10000 +BASELINE_FILE="" +SAVE_BASELINE="" +DB_URL="${DATABASE_URL:-}" BENCHMARK_DIR="benchmarks" TIMESTAMP=$(date +"%Y%m%d_%H%M%S") -RESULTS_FILE="${BENCHMARK_DIR}/results_${TIMESTAMP}.log" -# Colors for output +# ── colours ─────────────────────────────────────────────────────────────────── RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' -NC='\033[0m' # No Color +CYAN='\033[0;36m' +NC='\033[0m' -# Parse arguments +# ── argument parsing ────────────────────────────────────────────────────────── while [[ $# -gt 0 ]]; do case $1 in - --duration) - DURATION="$2" - shift 2 - ;; - --config) - CONFIG_FILE="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" - echo "Usage: $0 [--duration SECS] [--config CONFIG_FILE]" - exit 1 + --phase) PHASE="$2"; shift 2 ;; + --events) EVENTS="$2"; shift 2 ;; + --baseline) BASELINE_FILE="$2"; shift 2 ;; + --save-baseline) SAVE_BASELINE="$2"; shift 2 ;; + --db-url) DB_URL="$2"; shift 2 ;; + -h|--help) + sed -n '3,40p' "$0" | sed 's/^# //' | sed 's/^#//' + exit 0 ;; + *) echo "Unknown option: $1"; echo "Run $0 --help for usage."; exit 1 ;; esac done -# Create results directory -mkdir -p "$BENCHMARK_DIR" +# ── load .env if DATABASE_URL is still unset ───────────────────────────────── +if [[ -z "$DB_URL" && -f ".env" ]]; then + while IFS= read -r line; do + [[ "$line" =~ ^DATABASE_URL= ]] && DB_URL="${line#DATABASE_URL=}" && break + done < ".env" +fi -echo -e "${GREEN}Lumenqraph Indexer Throughput Benchmark${NC}" -echo "==========================================" -echo "Timestamp: $TIMESTAMP" -echo "Duration: ${DURATION}s" -echo "Config: $CONFIG_FILE" +# ── helpers ─────────────────────────────────────────────────────────────────── +header() { echo -e "\n${CYAN}══ $* ══${NC}"; } +ok() { echo -e " ${GREEN}✓${NC} $*"; } +warn() { echo -e " ${YELLOW}⚠${NC} $*"; } +fail() { echo -e " ${RED}✗${NC} $*"; } +section() { echo -e " ${YELLOW}→${NC} $*"; } + +# ── print banner ────────────────────────────────────────────────────────────── +echo "" +echo -e "${GREEN}╔══════════════════════════════════════════════════╗${NC}" +echo -e "${GREEN}║ Lumenqraph Indexer Benchmark (mock-RPC mode) ║${NC}" +echo -e "${GREEN}╚══════════════════════════════════════════════════╝${NC}" +echo " Timestamp : $TIMESTAMP" +echo " Phase : $PHASE" +echo " Events : $EVENTS" echo "" -# Check if config file exists -if [[ ! -f "$CONFIG_FILE" ]]; then - echo -e "${RED}Error: Config file not found: $CONFIG_FILE${NC}" - echo "Please copy .env.example to .env and configure DATABASE_URL and RPC_URL" +mkdir -p "$BENCHMARK_DIR" +RESULTS_FILE="${BENCHMARK_DIR}/results_${TIMESTAMP}.json" + +# ── build in release mode (criterion benches need release) ─────────────────── +header "Building (release)" +if cargo build --release -p lumenqraph-indexer 2>&1 | grep -E "^error" ; then + fail "Build failed — aborting" exit 1 fi +ok "Build succeeded" -# Load environment from config -export $(cat "$CONFIG_FILE" | grep -v '^#' | xargs) +# ── phase: xdr_decode ───────────────────────────────────────────────────────── +run_xdr_decode() { + header "Phase 1 / 3 — XDR decode throughput" + section "Running criterion benchmark: xdr_decode" + echo "" -# Verify required environment variables -if [[ -z "${DATABASE_URL:-}" ]]; then - echo -e "${RED}Error: DATABASE_URL not set in $CONFIG_FILE${NC}" - exit 1 -fi + # criterion writes machine-readable output to target/criterion// + cargo bench \ + --bench bench_indexer \ + -- xdr_decode \ + 2>&1 | grep -E "(xdr_decode|time|thrpt|ns/iter)" | head -20 -if [[ -z "${RPC_URL:-}" ]]; then - echo -e "${RED}Error: RPC_URL not set in $CONFIG_FILE${NC}" - exit 1 -fi + # Extract the mean from criterion's estimates.json if available + local est + est=$(find target/criterion -name "estimates.json" -path "*/xdr_decode/*" 2>/dev/null | head -1) + if [[ -n "$est" ]]; then + local mean_ns + mean_ns=$(python3 -c "import json,sys; d=json.load(open('$est')); print(d['mean']['point_estimate'])" 2>/dev/null || echo "N/A") + ok "Mean latency per event: ${mean_ns} ns" + fi +} -echo -e "${YELLOW}Configuration:${NC}" -echo " DATABASE_URL: ${DATABASE_URL}" -echo " RPC_URL: ${RPC_URL}" -echo " PAGE_SIZE: ${INDEXER_PAGE_SIZE:-100}" -echo " BATCH_SIZE: ${INDEXER_BATCH_SIZE:-100}" -echo " ENRICHMENT: ${ENRICHMENT_ENABLED:-true}" -echo "" +# ── phase: enrichment ───────────────────────────────────────────────────────── +run_enrichment() { + header "Phase 2 / 3 — Event enrichment throughput" + section "Running criterion benchmark: enrichment" + echo "" -# Check database connectivity -echo -e "${YELLOW}Checking database connectivity...${NC}" -if ! psql "$DATABASE_URL" -c "SELECT 1" > /dev/null 2>&1; then - echo -e "${RED}Error: Cannot connect to database${NC}" - echo "DATABASE_URL: $DATABASE_URL" - exit 1 -fi -echo -e "${GREEN}Database connected successfully${NC}" -echo "" + cargo bench \ + --bench bench_indexer \ + -- enrichment \ + 2>&1 | grep -E "(enrichment|time|thrpt|ns/iter)" | head -20 -# Check RPC connectivity -echo -e "${YELLOW}Checking RPC connectivity...${NC}" -if ! timeout 5 curl -s -X POST "$RPC_URL" \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc": "2.0", "id": 1, "method": "getHealth"}' | grep -q "healthy"; then - echo -e "${RED}Warning: RPC health check failed${NC}" - echo "The RPC endpoint may be unavailable or slow" -else - echo -e "${GREEN}RPC connected successfully${NC}" -fi -echo "" + local est + est=$(find target/criterion -name "estimates.json" -path "*/enrichment/*" 2>/dev/null | head -1) + if [[ -n "$est" ]]; then + local mean_ns + mean_ns=$(python3 -c "import json,sys; d=json.load(open('$est')); print(d['mean']['point_estimate'])" 2>/dev/null || echo "N/A") + ok "Mean latency per event: ${mean_ns} ns" + fi +} -# Run benchmark -echo -e "${YELLOW}Starting benchmark...${NC}" -echo "" +# ── phase: db_insert ───────────────────────────────────────────────────────── +run_db_insert() { + header "Phase 3 / 3 — Database insert throughput (mock-RPC end-to-end)" + if [[ -z "$DB_URL" ]]; then + warn "DATABASE_URL not set — skipping db_insert phase" + warn "Pass --db-url or set DATABASE_URL to run this phase" + return + fi -# Build in release mode if not already done -if ! cargo build --release -p lumenqraph-indexer 2>&1 | tail -3; then - echo -e "${RED}Failed to build indexer${NC}" - exit 1 -fi + section "Checking database connectivity" + if ! psql "$DB_URL" -c "SELECT 1" > /dev/null 2>&1; then + fail "Cannot connect to database: $DB_URL" + warn "Skipping db_insert phase" + return + fi + ok "Database connected" -# Run the indexer with timing instrumentation -# Note: This assumes the indexer binary is available at the expected path -# For now, we'll provide timing information via shell -START_TIME=$(date +%s%N) + section "Running end-to-end mock-RPC benchmark" + echo "" -# Run the indexer - capture logs and output -BENCHMARK_RUNS=$(cargo run --release -p lumenqraph-indexer 2>&1 || true) + # The Rust integration test suite contains the mock RPC + fetch_and_store + # path. We run it with a timing wrapper here. + # The criterion bench_indexer::db_insert benchmark handles its own + # pool setup and teardown — it uses TEST_DATABASE_URL. + TEST_DATABASE_URL="$DB_URL" cargo bench \ + --bench bench_indexer \ + -- db_insert \ + 2>&1 | grep -E "(db_insert|time|thrpt|ns/iter|events/s)" | head -20 -END_TIME=$(date +%s%N) -ELAPSED_MS=$(( (END_TIME - START_TIME) / 1000000 )) -ELAPSED_SECS=$(echo "scale=2; $ELAPSED_MS / 1000" | bc) + ok "DB insert phase complete" +} -echo "" -echo -e "${GREEN}Benchmark complete${NC}" -echo "Elapsed time: ${ELAPSED_SECS}s" -echo "" +# ── collect results ─────────────────────────────────────────────────────────── +collect_results() { + header "Collecting results" -# Save results -cat > "$RESULTS_FILE" << EOF -# Lumenqraph Indexer Benchmark Results -Timestamp: $TIMESTAMP -Duration: ${DURATION}s -Actual elapsed: ${ELAPSED_SECS}s - -## Configuration -DATABASE_URL: $DATABASE_URL -RPC_URL: $RPC_URL -PAGE_SIZE: ${INDEXER_PAGE_SIZE:-100} -BATCH_SIZE: ${INDEXER_BATCH_SIZE:-100} -ENRICHMENT_ENABLED: ${ENRICHMENT_ENABLED:-true} - -## Raw Output -$BENCHMARK_RUNS - -## Analysis -Please check the logs above for: -- Total events processed -- Throughput (events/sec) -- Average insert latency -- Average decode latency -- Peak memory usage -- Any errors or warnings -EOF - -echo -e "${GREEN}Results saved to: $RESULTS_FILE${NC}" -echo "" + local results="{}" + local date_str + date_str=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + # Parse criterion's estimates.json for each benchmark if present + for bench_name in xdr_decode enrichment db_insert; do + local est + est=$(find target/criterion -name "estimates.json" -path "*/${bench_name}/*" 2>/dev/null | head -1) + if [[ -n "$est" ]] && command -v python3 &>/dev/null; then + local mean_ns median_ns + mean_ns=$(python3 -c " +import json +d = json.load(open('$est')) +print(d['mean']['point_estimate']) +" 2>/dev/null || echo "null") + results=$(echo "$results" | python3 -c " +import json, sys +d = json.load(sys.stdin) +d['$bench_name'] = {'mean_ns': $mean_ns, 'timestamp': '$date_str', 'events': $EVENTS} +print(json.dumps(d, indent=2)) +" 2>/dev/null || echo "$results") + fi + done + + echo "$results" > "$RESULTS_FILE" + ok "Results saved to $RESULTS_FILE" + echo "$results" +} + +# ── baseline comparison ─────────────────────────────────────────────────────── +compare_baseline() { + local current_file="$1" + local baseline_file="$2" + + if ! command -v python3 &>/dev/null; then + warn "python3 not found — skipping baseline comparison" + return + fi + + header "Comparing against baseline: $baseline_file" + + python3 - "$current_file" "$baseline_file" <<'PYEOF' +import json, sys + +current = json.load(open(sys.argv[1])) +baseline = json.load(open(sys.argv[2])) + +REGRESSION_THRESHOLD = 0.10 # 10% +any_regression = False + +for bench in ("xdr_decode", "enrichment", "db_insert"): + if bench not in current or bench not in baseline: + print(f" ⚠ {bench}: not present in both runs — skipping") + continue + + cur_ns = current[bench]["mean_ns"] + base_ns = baseline[bench]["mean_ns"] + + if cur_ns is None or base_ns is None: + print(f" ⚠ {bench}: missing data — skipping") + continue + + delta_pct = (cur_ns - base_ns) / base_ns * 100 + direction = "slower" if delta_pct > 0 else "faster" + symbol = "✗" if delta_pct > REGRESSION_THRESHOLD * 100 else "✓" + + print(f" {symbol} {bench}: {cur_ns/1e6:.3f} ms (baseline {base_ns/1e6:.3f} ms, " + f"{abs(delta_pct):.1f}% {direction})") + + if delta_pct > REGRESSION_THRESHOLD * 100: + any_regression = True + print(f" REGRESSION: >{REGRESSION_THRESHOLD*100:.0f}% slower than baseline") + +if any_regression: + print("\nOne or more benchmarks regressed by more than 10%.") + sys.exit(1) +else: + print("\nAll benchmarks within acceptable bounds (< 10% regression).") +PYEOF +} + +# ── main ────────────────────────────────────────────────────────────────────── +case "$PHASE" in + all) + run_xdr_decode + run_enrichment + run_db_insert + ;; + xdr_decode) run_xdr_decode ;; + enrichment) run_enrichment ;; + db_insert) run_db_insert ;; + *) + fail "Unknown phase: $PHASE (must be all | xdr_decode | enrichment | db_insert)" + exit 1 + ;; +esac + +RESULTS=$(collect_results) + +if [[ -n "$SAVE_BASELINE" ]]; then + cp "$RESULTS_FILE" "$SAVE_BASELINE" + ok "Baseline saved to $SAVE_BASELINE" +fi -# Parse and display key metrics if available -if echo "$BENCHMARK_RUNS" | grep -q "events processed"; then - echo -e "${YELLOW}Key Metrics:${NC}" - echo "$BENCHMARK_RUNS" | grep -E "events processed|Throughput|latency" || true +if [[ -n "$BASELINE_FILE" ]]; then + compare_baseline "$RESULTS_FILE" "$BASELINE_FILE" fi echo "" -echo "For detailed analysis, see: $RESULTS_FILE" +echo -e "${GREEN}Benchmark complete.${NC} Results: $RESULTS_FILE" diff --git a/scripts/e2e_smoke_test.sh b/scripts/e2e_smoke_test.sh new file mode 100644 index 0000000..fe6989f --- /dev/null +++ b/scripts/e2e_smoke_test.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# End-to-end smoke test for Lumenqraph full stack. +# Verifies that the indexer → database → API pipeline works correctly. +# Expects the full stack to be running (via docker-compose). + +set -euo pipefail + +API_URL="${API_URL:-http://localhost:8080}" +MAX_ATTEMPTS=30 +RETRY_DELAY_SECS=2 +INDEXER_URL="${INDEXER_URL:-http://localhost:9090}" + +echo "Starting E2E smoke test..." +echo "API URL: $API_URL" +echo "Max attempts: $MAX_ATTEMPTS" + +# Wait for API health endpoint to report ok +attempt=0 +while [ $attempt -lt $MAX_ATTEMPTS ]; do + if curl -sf "$API_URL/health" > /dev/null 2>&1; then + echo "✓ API is healthy" + break + fi + attempt=$((attempt + 1)) + echo "Waiting for API health... ($attempt/$MAX_ATTEMPTS)" + sleep $RETRY_DELAY_SECS +done + +if [ $attempt -eq $MAX_ATTEMPTS ]; then + echo "✗ API failed to become healthy after $((MAX_ATTEMPTS * RETRY_DELAY_SECS)) seconds" + exit 1 +fi + +# Query /contracts endpoint to verify API is responding +echo "Testing API endpoints..." +if ! contracts=$(curl -sf "$API_URL/contracts" 2>/dev/null); then + echo "✗ Failed to fetch /contracts endpoint" + exit 1 +fi + +echo "✓ /contracts endpoint responds" + +# Parse the response to check if it's valid JSON +if ! echo "$contracts" | jq empty 2>/dev/null; then + echo "✗ /contracts response is not valid JSON" + exit 1 +fi + +echo "✓ /contracts response is valid JSON" + +# Check if we have any contracts (this is optional, but helpful for debugging) +contract_count=$(echo "$contracts" | jq 'length // 0' 2>/dev/null || echo 0) +echo "Total contracts in database: $contract_count" + +# Test /health endpoint returns expected fields +echo "Testing /health endpoint..." +if ! health=$(curl -sf "$API_URL/health" 2>/dev/null); then + echo "✗ Failed to fetch /health endpoint" + exit 1 +fi + +if ! echo "$health" | jq -e '.status' > /dev/null 2>/dev/null; then + echo "✗ /health response missing 'status' field" + exit 1 +fi + +health_status=$(echo "$health" | jq -r '.status' 2>/dev/null || echo "unknown") +echo "✓ API health status: $health_status" + +echo "" +echo "✅ E2E smoke test passed! Full pipeline is operational." +exit 0 diff --git a/scripts/gen_api_key.sh b/scripts/gen_api_key.sh index 58e530a..37c1bd7 100755 --- a/scripts/gen_api_key.sh +++ b/scripts/gen_api_key.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Generate an API key, store only its SHA-256 hash, and print the key once. -# Usage: DATABASE_URL=... ./scripts/gen_api_key.sh [name] [tier] [rate_per_min] +# Usage: ./scripts/gen_api_key.sh [name] [tier] [rate_per_min] +# Environment: DATABASE_URL must be set (recommended: via .pgpass for secure password handling) set -euo pipefail NAME="${1:-default}" @@ -11,6 +12,12 @@ LIMIT="${3:-60}" KEY="lqk_$(head -c 24 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9' | head -c 32)" HASH=$(printf '%s' "$KEY" | sha256sum | cut -d' ' -f1) +# Use PGPASSWORD for password handling instead of embedding in DATABASE_URL. +# This keeps credentials out of process listings (visible via ps/proc). +# For production, use a .pgpass file: ~/.pgpass with mode 0600 containing: +# hostname:port:database:username:password +export PGPASSWORD="${PGPASSWORD:-}" + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c \ "INSERT INTO api_keys (key_hash, name, tier, rate_limit_per_min) VALUES ('$HASH', '$NAME', '$TIER', $LIMIT)" diff --git a/scripts/import_contracts.py b/scripts/import_contracts.py index dd59577..82750e5 100644 --- a/scripts/import_contracts.py +++ b/scripts/import_contracts.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Import a list of Soroban contract ids from Stellar.Expert. +"""Import a list of Soroban contract ids from Stellar.Expert and optionally import into Lumenqraph. Stellar.Expert already tracks every deployed contract; this pulls a batch of them so Lumenqraph can index them. Prints a comma-separated list suitable for @@ -17,17 +17,24 @@ [--order desc|asc] [--limit N] [--include-empty] + [--api-url ] Examples: # 20 mainnet contracts that have emitted events (good for a demo): python3 scripts/import_contracts.py --network public --limit 20 # include contracts with zero events too: python3 scripts/import_contracts.py --include-empty --limit 20 + # import into local Lumenqraph instance: + python3 scripts/import_contracts.py --api-url http://localhost:8080 --limit 20 """ import argparse import json +import re import sys +import time +import urllib.error import urllib.request +from typing import Optional API = "https://api.stellar.expert/explorer" @@ -42,9 +49,46 @@ PAGE_SIZE = 200 # Guard against scanning the whole ledger when few contracts are active. MAX_PAGES = 25 +# Retry configuration +MAX_RETRIES = 3 +INITIAL_BACKOFF_SECS = 1 +REQUEST_TIMEOUT_SECS = 30 + +# Contract ID format validation: Stellar contract IDs start with 'C' followed by alphanumeric +CONTRACT_ID_PATTERN = re.compile(r"^C[A-Z2-7]{55}$") + + +def validate_contract_id(contract_id: str) -> bool: + """Validate that a contract ID has the correct format (C-strkey).""" + return CONTRACT_ID_PATTERN.match(contract_id) is not None + + +def fetch_with_retry(url: str, max_retries: int = MAX_RETRIES) -> dict: + """Fetch a URL with exponential backoff retry logic.""" + for attempt in range(max_retries): + try: + req = urllib.request.Request(url, headers=HEADERS) + with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SECS) as resp: + return json.load(resp) + except urllib.error.HTTPError as e: + if e.code >= 500 and attempt < max_retries - 1: + backoff = INITIAL_BACKOFF_SECS * (2**attempt) + print(f"HTTP {e.code} on {url}, retrying in {backoff}s...", file=sys.stderr) + time.sleep(backoff) + continue + raise + except urllib.error.URLError as e: + if attempt < max_retries - 1: + backoff = INITIAL_BACKOFF_SECS * (2**attempt) + print(f"Network error: {e.reason}, retrying in {backoff}s...", file=sys.stderr) + time.sleep(backoff) + continue + raise + raise RuntimeError(f"Failed to fetch {url} after {max_retries} attempts") def fetch(network: str, order: str, limit: int, active_only: bool) -> list[dict]: + """Fetch contracts from Stellar.Expert with error handling.""" out: list[dict] = [] seen: set[str] = set() cursor = None @@ -54,9 +98,11 @@ def fetch(network: str, order: str, limit: int, active_only: bool) -> list[dict] url = f"{API}/{network}/contract?order={order}&limit={PAGE_SIZE}" if cursor: url += f"&cursor={cursor}" - req = urllib.request.Request(url, headers=HEADERS) - with urllib.request.urlopen(req, timeout=30) as resp: - data = json.load(resp) + try: + data = fetch_with_retry(url) + except Exception as e: + print(f"error fetching contracts: {e}", file=sys.stderr) + raise records = data.get("_embedded", {}).get("records", []) if not records: break @@ -78,6 +124,29 @@ def fetch(network: str, order: str, limit: int, active_only: bool) -> list[dict] return out[:limit] +def import_to_lumenqraph(api_url: str, contract_ids: list[str]) -> tuple[int, list[str]]: + """Import contract IDs into Lumenqraph API. Returns (success_count, failed_ids).""" + failed_ids = [] + for contract_id in contract_ids: + if not validate_contract_id(contract_id): + print(f"invalid contract ID format: {contract_id}", file=sys.stderr) + failed_ids.append(contract_id) + continue + + url = f"{api_url}/contracts/{contract_id}" + try: + req = urllib.request.Request(url, method="POST", headers={"content-type": "application/json"}) + with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SECS) as resp: + if resp.status not in (200, 201, 204): + print(f"failed to import {contract_id}: HTTP {resp.status}", file=sys.stderr) + failed_ids.append(contract_id) + except Exception as e: + print(f"failed to import {contract_id}: {e}", file=sys.stderr) + failed_ids.append(contract_id) + + return len(contract_ids) - len(failed_ids), failed_ids + + def main() -> int: ap = argparse.ArgumentParser(description="Import Soroban contract ids from Stellar.Expert") ap.add_argument("--network", default="public", choices=["public", "testnet"]) @@ -93,11 +162,17 @@ def main() -> int: action="store_true", help="also include contracts with zero events (default: active only)", ) + ap.add_argument( + "--api-url", + type=str, + default=None, + help="if provided, import fetched contracts into this Lumenqraph API URL", + ) args = ap.parse_args() try: records = fetch(args.network, args.order, args.limit, not args.include_empty) - except Exception as e: # noqa: BLE001 + except Exception as e: print(f"error: {e}", file=sys.stderr) return 1 @@ -107,14 +182,24 @@ def main() -> int: ids = [r["contract"] for r in records] total_events = sum(r.get("events") or 0 for r in records) - # Human-readable summary to stderr; the machine-usable list to stdout. scope = "all" if args.include_empty else "active" print( f"fetched {len(ids)} {args.network} contracts ({scope}, " f"{total_events} events total)", file=sys.stderr, ) - print(",".join(ids)) + + # Import to Lumenqraph if API URL provided + if args.api_url: + success_count, failed_ids = import_to_lumenqraph(args.api_url, ids) + print(f"imported {success_count}/{len(ids)} contracts to {args.api_url}", file=sys.stderr) + if failed_ids: + print(f"failed imports: {', '.join(failed_ids)}", file=sys.stderr) + return 1 + else: + # Output CSV list to stdout for use in env vars + print(",".join(ids)) + return 0 diff --git a/scripts/run-all-in-one.sh b/scripts/run-all-in-one.sh index 4e19ec5..ce82bd7 100755 --- a/scripts/run-all-in-one.sh +++ b/scripts/run-all-in-one.sh @@ -24,6 +24,30 @@ if [[ -n "${PORT:-}" && -z "${API_BIND_ADDR:-}" ]]; then export API_BIND_ADDR="0.0.0.0:${PORT}" fi +# ---- Run migrations FIRST --------------------------------------------------- +# The indexer would eventually run migrations itself, but the API and webhook +# services start concurrently and may attempt database connections before the +# schema exists on a fresh database. Running `sqlx migrate run` here makes +# migrations an explicit prerequisite: if this step fails the whole container +# exits immediately rather than starting a partially-configured stack. +echo "run-all-in-one: running database migrations…" >&2 +if ! sqlx migrate run --database-url "${DATABASE_URL}"; then + echo "run-all-in-one: migrations FAILED — aborting startup" >&2 + exit 1 +fi +echo "run-all-in-one: migrations complete" >&2 + +# Run migrations for the optional testnet database as well, before any of its +# services start. +if [[ -n "${TESTNET_DATABASE_URL:-}" ]]; then + echo "run-all-in-one: running testnet database migrations…" >&2 + if ! sqlx migrate run --database-url "${TESTNET_DATABASE_URL}"; then + echo "run-all-in-one: testnet migrations FAILED — aborting startup" >&2 + exit 1 + fi + echo "run-all-in-one: testnet migrations complete" >&2 +fi + pids=() # ---- Optional testnet pair (must start before the public API so it can be @@ -54,7 +78,9 @@ if [[ -n "${TESTNET_DATABASE_URL:-}" ]]; then fi # ---- Primary pair ------------------------------------------------------------ -lumenqraph-indexer & +# Migrations have already been applied above; pass SKIP_MIGRATIONS=true so the +# indexer does not attempt to run them a second time. +SKIP_MIGRATIONS=true lumenqraph-indexer & pids+=($!) lumenqraph-api & pids+=($!) diff --git a/scripts/validate_dashboard_metrics.py b/scripts/validate_dashboard_metrics.py new file mode 100644 index 0000000..d07ed4b --- /dev/null +++ b/scripts/validate_dashboard_metrics.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Validate that Grafana dashboard metrics match those defined in Rust source code. + +This script ensures dashboard panels reference only metrics that actually exist +in the codebase, catching drift early. +""" +import json +import re +import sys +from pathlib import Path + + +def extract_metrics_from_dashboard(dashboard_path: str) -> set[str]: + """Extract all metric names referenced in Grafana dashboard.""" + with open(dashboard_path) as f: + dashboard = json.load(f) + + metrics = set() + for panel in dashboard.get("panels", []): + for target in panel.get("targets", []): + expr = target.get("expr", "") + if expr: + # Extract metric names from Prometheus expressions + # Patterns: metric_name, rate(metric_name), histogram_quantile(...metric_name...) + found = re.findall(r"\b(lumenqraph_\w+)\b", expr) + metrics.update(found) + + return metrics + + +def extract_metrics_from_rust(rust_dir: str) -> set[str]: + """Extract all metric names defined in Rust source code.""" + metrics = set() + + for rust_file in Path(rust_dir).rglob("metrics.rs"): + with open(rust_file) as f: + content = f.read() + # Find all metric definitions: "lumenqraph_metric_name" + found = re.findall(r'"(lumenqraph_\w+)"', content) + metrics.update(found) + + return metrics + + +def main() -> int: + repo_root = Path(__file__).parent.parent + dashboard_path = repo_root / "monitoring" / "grafana_dashboard.json" + rust_crates = repo_root / "crates" + + if not dashboard_path.exists(): + print(f"error: dashboard not found at {dashboard_path}", file=sys.stderr) + return 1 + + try: + dashboard_metrics = extract_metrics_from_dashboard(str(dashboard_path)) + rust_metrics = extract_metrics_from_rust(str(rust_crates)) + except Exception as e: + print(f"error reading files: {e}", file=sys.stderr) + return 1 + + # Find metrics in dashboard but not in code + missing = dashboard_metrics - rust_metrics + if missing: + print("error: dashboard references metrics not found in code:", file=sys.stderr) + for metric in sorted(missing): + print(f" - {metric}", file=sys.stderr) + return 1 + + # Find metrics in code but not in dashboard (warning only) + unused = rust_metrics - dashboard_metrics + if unused: + print("warning: metrics in code but not in dashboard:", file=sys.stderr) + for metric in sorted(unused): + print(f" - {metric}", file=sys.stderr) + + print(f"✓ dashboard references {len(dashboard_metrics)} valid metrics", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sdk/python/lumenqraph/__init__.py b/sdk/python/lumenqraph/__init__.py index 698fec7..7bf5b02 100644 --- a/sdk/python/lumenqraph/__init__.py +++ b/sdk/python/lumenqraph/__init__.py @@ -1,14 +1,27 @@ """ Lumenqraph Python SDK — a typed client over the Lumenqraph REST + GraphQL API. -Example usage: +Synchronous example:: from lumenqraph import LumenqraphClient lq = LumenqraphClient(base_url="http://localhost:8080") contracts = lq.list_contracts() - for event in lq.paginate_events(contracts[0]["contract_id"]): + for event in lq.paginate_events(contracts["data"][0]["contract_id"]): print(event["event_name"], event.get("enriched") or event.get("decoded_value")) + +Async example:: + + import asyncio + from lumenqraph import AsyncLumenqraphClient + + async def main(): + async with AsyncLumenqraphClient(base_url="http://localhost:8080") as lq: + contracts = await lq.list_contracts() + async for event in lq.paginate_events(contracts["data"][0]["contract_id"]): + print(event["event_name"], event.get("enriched") or event.get("decoded_value")) + + asyncio.run(main()) """ from .client import ( @@ -17,10 +30,12 @@ ClientOptions, RetryOptions, ) +from .async_client import AsyncLumenqraphClient from .webhook import verify_webhook_signature __all__ = [ "LumenqraphClient", + "AsyncLumenqraphClient", "LumenqraphError", "ClientOptions", "RetryOptions", diff --git a/sdk/python/lumenqraph/async_client.py b/sdk/python/lumenqraph/async_client.py new file mode 100644 index 0000000..4232e0b --- /dev/null +++ b/sdk/python/lumenqraph/async_client.py @@ -0,0 +1,367 @@ +"""Async (asyncio) Lumenqraph API client. + +This module mirrors the surface of :class:`lumenqraph.client.LumenqraphClient` +but uses :mod:`asyncio` and :class:`urllib.request` via a thread-pool executor +(so no third-party dependency is needed) for all HTTP I/O. + +For high-concurrency use-cases you may prefer to install ``aiohttp`` or +``httpx`` and wrap them instead — this implementation favours zero dependencies +over raw throughput. + +Example:: + + import asyncio + from lumenqraph.async_client import AsyncLumenqraphClient + + async def main(): + async with AsyncLumenqraphClient(base_url="http://localhost:8080") as lq: + contracts = await lq.list_contracts() + async for event in lq.paginate_events(contracts["data"][0]["contract_id"]): + print(event["event_name"], event.get("enriched") or event.get("decoded_value")) + + asyncio.run(main()) +""" + +import asyncio +import json +import time +import urllib.error +import urllib.request +from typing import Any, AsyncGenerator, Dict, Optional +from urllib.parse import urlencode + +from .client import LumenqraphError, RetryOptions + + +class AsyncLumenqraphClient: + """Async Lumenqraph API client. + + All methods are coroutines. HTTP I/O is dispatched to a thread-pool + executor so the event loop is never blocked. + + The client can be used as an async context manager:: + + async with AsyncLumenqraphClient(base_url="http://localhost:8080") as lq: + health = await lq.health() + + It can also be instantiated directly without the context-manager protocol; + call :meth:`aclose` explicitly when done if you need deterministic cleanup. + """ + + DEFAULT_MAX_RETRIES = 3 + DEFAULT_BASE_DELAY_MS = 250 + DEFAULT_MAX_DELAY_MS = 30_000 + DEFAULT_TIMEOUT_MS = 10_000 + RETRYABLE_STATUSES = {429, 502, 503, 504} + + def __init__( + self, + base_url: str, + api_key: Optional[str] = None, + retry: Optional[RetryOptions] = None, + ) -> None: + """Initialise the async Lumenqraph client. + + Args: + base_url: Base URL of the Lumenqraph API (e.g. ``http://localhost:8080``). + api_key: Optional API key for authenticated requests. + retry: Optional retry / timeout policy. + """ + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self._retry = self._merge_retry(retry or {}) + + def _merge_retry(self, opts: RetryOptions) -> Dict[str, int]: + return { + "max_retries": opts.get("max_retries", self.DEFAULT_MAX_RETRIES), + "base_delay_ms": opts.get("base_delay_ms", self.DEFAULT_BASE_DELAY_MS), + "max_delay_ms": opts.get("max_delay_ms", self.DEFAULT_MAX_DELAY_MS), + "timeout_ms": opts.get("timeout_ms", self.DEFAULT_TIMEOUT_MS), + } + + async def __aenter__(self) -> "AsyncLumenqraphClient": + return self + + async def __aexit__(self, *_: Any) -> None: + await self.aclose() + + async def aclose(self) -> None: + """Release any resources held by the client (currently a no-op).""" + + # ---- Core request machinery ---- + + def _make_url(self, path: str, query: Optional[Dict[str, Any]] = None) -> str: + url = self.base_url + path + if query: + filtered = {k: v for k, v in query.items() if v is not None} + if filtered: + url = f"{url}?{urlencode(filtered)}" + return url + + def _sync_request( + self, + method: str, + url: str, + body: Optional[bytes], + headers: Dict[str, str], + timeout: float, + ) -> Any: + """Blocking HTTP call — run in a thread-pool executor.""" + req = urllib.request.Request(url, data=body, headers=headers, method=method) + attempt = 0 + max_retries = self._retry["max_retries"] + + while True: + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + text = resp.read().decode("utf-8") + return json.loads(text) if text else None + except urllib.error.HTTPError as exc: + status = exc.code + text = exc.read().decode("utf-8") + try: + body_obj = json.loads(text) + except json.JSONDecodeError: + body_obj = text + + if status not in self.RETRYABLE_STATUSES or attempt >= max_retries: + raise LumenqraphError(f"HTTP {status}: {text}", status, body_obj) + + delay_s = min( + self._retry["base_delay_ms"] * (2 ** attempt), + self._retry["max_delay_ms"], + ) / 1000.0 + time.sleep(delay_s) + attempt += 1 + + except urllib.error.URLError as exc: + if attempt >= max_retries: + raise LumenqraphError(f"Network error: {exc}", 0, None) + delay_s = min( + self._retry["base_delay_ms"] * (2 ** attempt), + self._retry["max_delay_ms"], + ) / 1000.0 + time.sleep(delay_s) + attempt += 1 + + async def _request( + self, + method: str, + path: str, + query: Optional[Dict[str, Any]] = None, + body: Optional[Dict[str, Any]] = None, + ) -> Any: + url = self._make_url(path, query) + headers: Dict[str, str] = {"Content-Type": "application/json"} + if self.api_key: + headers["x-api-key"] = self.api_key + + encoded_body: Optional[bytes] = None + if body is not None: + encoded_body = json.dumps(body).encode("utf-8") + + timeout = self._retry["timeout_ms"] / 1000.0 + + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + None, + self._sync_request, + method, + url, + encoded_body, + headers, + timeout, + ) + + async def _get(self, path: str, query: Optional[Dict[str, Any]] = None) -> Any: + return await self._request("GET", path, query) + + async def _post(self, path: str, body: Optional[Dict[str, Any]] = None) -> Any: + return await self._request("POST", path, body=body) + + # ---- Public API surface (mirrors LumenqraphClient) ---- + + async def health(self) -> Dict[str, Any]: + """Get health and indexing-lag report.""" + return await self._get("/health") + + async def list_contracts( + self, limit: int = 200, offset: int = 0 + ) -> Dict[str, Any]: + """Get contracts with pagination.""" + return await self._get("/contracts", {"limit": limit, "offset": offset}) + + async def get_interface( + self, contract_id: str, version: Optional[int] = None + ) -> Dict[str, Any]: + """Get a contract's decoded on-chain interface.""" + query: Dict[str, Any] = {} + if version is not None: + query["version"] = version + return await self._get(f"/contracts/{contract_id}/interface", query) + + async def get_state( + self, contract_id: str, limit: int = 1 + ) -> Dict[str, Any]: + """Get versioned instance-storage snapshots, newest first.""" + return await self._get( + f"/contracts/{contract_id}/state", {"limit": limit} + ) + + async def get_data( + self, + contract_id: str, + label: Optional[str] = None, + limit: int = 100, + ) -> Dict[str, Any]: + """Get latest value of every per-key entry.""" + query: Dict[str, Any] = {"limit": limit} + if label: + query["label"] = label + return await self._get(f"/contracts/{contract_id}/data", query) + + async def get_data_key( + self, contract_id: str, key_hash: str, limit: int = 50 + ) -> Dict[str, Any]: + """Get version history of a single per-key entry.""" + return await self._get( + f"/contracts/{contract_id}/data/{key_hash}", {"limit": limit} + ) + + async def list_events( + self, + contract_id: str, + limit: int = 50, + offset: int = 0, + event_name: Optional[str] = None, + after: Optional[str] = None, + ) -> Dict[str, Any]: + """Get recent events for a contract, newest first.""" + query: Dict[str, Any] = {"limit": limit} + if offset: + query["offset"] = offset + if event_name: + query["event_name"] = event_name + if after: + query["after"] = after + return await self._get(f"/contracts/{contract_id}/events", query) + + async def list_transfers( + self, + contract_id: Optional[str] = None, + limit: int = 50, + offset: int = 0, + ) -> Dict[str, Any]: + """Get materialized SEP-41 transfers.""" + path = f"/contracts/{contract_id}/transfers" if contract_id else "/transfers" + return await self._get(path, {"limit": limit, "offset": offset}) + + async def list_functions(self, contract_id: str) -> Dict[str, Any]: + """Get a contract's callable view functions and their typed signatures.""" + return await self._get(f"/contracts/{contract_id}/functions") + + async def call( + self, + contract_id: str, + function: str, + args: Optional[Any] = None, + source_account: Optional[str] = None, + ) -> Dict[str, Any]: + """Invoke a view function read-only and get a typed result.""" + return await self._post( + f"/contracts/{contract_id}/call", + {"function": function, "args": args, "source_account": source_account}, + ) + + async def simulate( + self, + contract_id: str, + function: str, + args: Optional[Any] = None, + source_account: Optional[str] = None, + ) -> Dict[str, Any]: + """Dry-run any call and preview its result, emitted events, and cost.""" + return await self._post( + f"/contracts/{contract_id}/simulate", + {"function": function, "args": args, "source_account": source_account}, + ) + + async def graphql( + self, + query: str, + variables: Optional[Dict[str, Any]] = None, + ) -> Any: + """Execute a raw GraphQL query against ``/graphql``.""" + body = await self._post("/graphql", {"query": query, "variables": variables or {}}) + errors = (body or {}).get("errors") + if errors: + messages = "; ".join(e.get("message", str(e)) for e in errors) + raise LumenqraphError(f"GraphQL error: {messages}", 200, errors) + return (body or {}).get("data") + + async def paginate_events( + self, + contract_id: str, + event_name: Optional[str] = None, + page_size: int = 100, + ) -> AsyncGenerator[Dict[str, Any], None]: + """Async generator over all events for a contract via cursor pagination. + + Transparently fetches page after page until all events have been + yielded. + + Args: + contract_id: Contract to fetch events for. + event_name: Optional event-name filter. + page_size: Number of events to request per page (default 100). + + Yields: + Event record dicts. + + Example:: + + async for event in lq.paginate_events(contract_id, event_name="transfer"): + print(event["ledger"], event.get("enriched")) + """ + cursor: Optional[str] = None + while True: + response = await self.list_events( + contract_id, + limit=page_size, + event_name=event_name, + after=cursor, + ) + data = response.get("data", []) + for event in data: + yield event + if not response.get("has_more", False): + break + cursor = response.get("next_cursor") + if not cursor: + break + + async def paginate_transfers( + self, + contract_id: Optional[str] = None, + page_size: int = 100, + ) -> AsyncGenerator[Dict[str, Any], None]: + """Async generator over all materialized transfers. + + Args: + contract_id: Optional contract to scope the query to. + page_size: Transfers per page (default 100). + + Yields: + Transfer record dicts. + """ + offset = 0 + while True: + response = await self.list_transfers( + contract_id, limit=page_size, offset=offset + ) + items = response if isinstance(response, list) else response.get("data", []) + for item in items: + yield item + if not items or len(items) < page_size: + break + offset += len(items) diff --git a/sdk/python/lumenqraph/webhook.py b/sdk/python/lumenqraph/webhook.py index 8e2a1c9..0ccfe51 100644 --- a/sdk/python/lumenqraph/webhook.py +++ b/sdk/python/lumenqraph/webhook.py @@ -1,42 +1,72 @@ -"""Webhook signature verification utilities.""" +"""Webhook signature verification utilities. + +The Lumenqraph server signs the raw request body with the subscription secret +and sends the result as:: + + X-Lumenqraph-Signature: sha256= + +This module provides :func:`verify_webhook_signature` to validate that header +in constant time using :mod:`hmac` from the Python standard library. +""" import hmac import hashlib -from typing import Optional -def verify_webhook_signature(body: str, signature: str, secret: str) -> bool: - """Verify a webhook signature. +def verify_webhook_signature( + body: str, + signature_header: str, + secret: str, +) -> bool: + """Verify a Lumenqraph webhook delivery signature. + + The server computes ``HMAC-SHA256(secret, raw_body)`` and sends the result + as ``X-Lumenqraph-Signature: sha256=``. Pass that full header value + (including the ``sha256=`` prefix) as *signature_header*. + + Comparison is performed in constant time via :func:`hmac.compare_digest` + so this function is safe to use in security-sensitive contexts. It mirrors + the server-side ``verify_hmac_signature()`` in + ``lumenqraph-core/src/crypto.rs`` and the ``verifyWebhook`` helper in the + TypeScript SDK. Args: - body: The raw webhook body as a string - signature: The signature from the X-Webhook-Signature header - secret: The webhook secret (from the webhook configuration) + body: Raw HTTP request body as a string. + signature_header: Value of the ``X-Lumenqraph-Signature`` header, + e.g. ``"sha256=abcdef…"``. + secret: The subscription secret returned at creation time. Returns: - True if the signature is valid, False otherwise + ``True`` if the signature is valid, ``False`` otherwise. - Example: - def webhook_handler(request): - signature = request.headers.get('X-Webhook-Signature') - body = request.get_data(as_text=True) - secret = "your-webhook-secret" + Example:: - if not verify_webhook_signature(body, signature, secret): - return {"error": "Invalid signature"}, 401 + from lumenqraph import verify_webhook_signature - # Process webhook - return {"ok": True}, 200 + # Flask example + @app.route("/hook", methods=["POST"]) + def webhook(): + sig = request.headers.get("X-Lumenqraph-Signature", "") + body = request.get_data(as_text=True) + if not verify_webhook_signature(body, sig, WEBHOOK_SECRET): + return {"error": "invalid signature"}, 401 + # process payload … + return {}, 200 """ - if not signature or not secret: + if not signature_header or not secret: return False - # Compute HMAC-SHA256 of the body with the secret - computed = hmac.new( + prefix = "sha256=" + if not signature_header.startswith(prefix): + return False + + provided_hex = signature_header[len(prefix):] + + computed_hex = hmac.new( secret.encode("utf-8"), body.encode("utf-8"), - hashlib.sha256 + hashlib.sha256, ).hexdigest() - # Use constant-time comparison to prevent timing attacks - return hmac.compare_digest(computed, signature) + # Constant-time comparison prevents timing-oracle attacks. + return hmac.compare_digest(computed_hex, provided_hex) diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml new file mode 100644 index 0000000..2e0f78f --- /dev/null +++ b/sdk/python/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.backends.legacy:build" + +[project] +name = "lumenqraph" +version = "0.1.0" +description = "Lumenqraph Python SDK — a typed client over the Lumenqraph REST + GraphQL API" +readme = "README.md" +license = { file = "LICENSE" } +authors = [{ name = "Lumen Scribe", email = "dev@lumenscribe.com" }] +requires-python = ">=3.8" +keywords = ["stellar", "soroban", "blockchain", "dapp", "lumenqraph"] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet", +] +# Zero runtime dependencies — stdlib only (urllib, asyncio, hmac, hashlib). +dependencies = [] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", + "pytest-asyncio>=0.23", + "mypy>=1.5", +] + +[project.urls] +Homepage = "https://github.com/Lumen-Scribe/Lumenqraph" +Documentation = "https://github.com/Lumen-Scribe/Lumenqraph#python-sdk" +"Source Code" = "https://github.com/Lumen-Scribe/Lumenqraph/tree/main/sdk/python" +"Bug Reports" = "https://github.com/Lumen-Scribe/Lumenqraph/issues" + +[tool.setuptools.packages.find] +where = ["."] +include = ["lumenqraph*"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.mypy] +python_version = "3.8" +strict = true +files = ["lumenqraph"] diff --git a/sdk/python/tests/test_async_client.py b/sdk/python/tests/test_async_client.py new file mode 100644 index 0000000..cefd9f9 --- /dev/null +++ b/sdk/python/tests/test_async_client.py @@ -0,0 +1,140 @@ +"""Unit tests for the async Lumenqraph client. + +These tests exercise the client's URL-building and pagination logic without +making real network calls by monkey-patching the internal ``_sync_request`` +method. +""" + +import asyncio +import unittest +from typing import Any, Dict +from unittest.mock import patch, MagicMock + +from lumenqraph import AsyncLumenqraphClient, LumenqraphError + + +class TestAsyncClientUrlBuilding(unittest.IsolatedAsyncioTestCase): + """Verify that query parameters are assembled correctly.""" + + async def test_list_contracts_default_params(self): + captured: Dict[str, Any] = {} + + async def mock_request(method, path, query=None, body=None): + captured["method"] = method + captured["path"] = path + captured["query"] = query + return {"data": [], "has_more": False} + + lq = AsyncLumenqraphClient(base_url="http://test") + with patch.object(lq, "_request", side_effect=mock_request): + await lq.list_contracts() + + self.assertEqual(captured["method"], "GET") + self.assertEqual(captured["path"], "/contracts") + self.assertEqual(captured["query"], {"limit": 200, "offset": 0}) + + async def test_list_events_with_event_name(self): + captured: Dict[str, Any] = {} + + async def mock_request(method, path, query=None, body=None): + captured["query"] = query + return {"data": [], "has_more": False, "next_cursor": None} + + lq = AsyncLumenqraphClient(base_url="http://test") + with patch.object(lq, "_request", side_effect=mock_request): + await lq.list_events("C1", event_name="transfer") + + self.assertIn("event_name", captured["query"]) + self.assertEqual(captured["query"]["event_name"], "transfer") + + async def test_call_sends_correct_body(self): + captured: Dict[str, Any] = {} + + async def mock_request(method, path, query=None, body=None): + captured["body"] = body + return {"contract_id": "C1", "function": "balance", + "result": {"type": "i128", "value": "0"}, + "simulated_at_ledger": 1} + + lq = AsyncLumenqraphClient(base_url="http://test") + with patch.object(lq, "_request", side_effect=mock_request): + await lq.call("C1", function="balance", args={"id": "G1"}) + + self.assertEqual(captured["body"]["function"], "balance") + self.assertEqual(captured["body"]["args"], {"id": "G1"}) + + +class TestAsyncClientPagination(unittest.IsolatedAsyncioTestCase): + """Verify cursor-based pagination exhausts all pages.""" + + async def test_paginate_events_follows_cursors(self): + pages = [ + {"data": [{"event_id": "e1"}, {"event_id": "e2"}], + "has_more": True, "next_cursor": "cur1"}, + {"data": [{"event_id": "e3"}], + "has_more": False, "next_cursor": None}, + ] + call_count = 0 + + async def mock_request(method, path, query=None, body=None): + nonlocal call_count + result = pages[call_count] + call_count += 1 + return result + + lq = AsyncLumenqraphClient(base_url="http://test") + with patch.object(lq, "_request", side_effect=mock_request): + events = [] + async for ev in lq.paginate_events("C1"): + events.append(ev) + + self.assertEqual(len(events), 3) + self.assertEqual(events[0]["event_id"], "e1") + self.assertEqual(events[2]["event_id"], "e3") + self.assertEqual(call_count, 2) + + async def test_paginate_events_single_page(self): + async def mock_request(method, path, query=None, body=None): + return {"data": [{"event_id": "e1"}], "has_more": False, "next_cursor": None} + + lq = AsyncLumenqraphClient(base_url="http://test") + with patch.object(lq, "_request", side_effect=mock_request): + events = [ev async for ev in lq.paginate_events("C1")] + + self.assertEqual(len(events), 1) + + +class TestAsyncClientErrors(unittest.IsolatedAsyncioTestCase): + """Verify error propagation from the underlying sync request.""" + + async def test_raises_lumenqraph_error_on_404(self): + async def mock_request(method, path, query=None, body=None): + raise LumenqraphError("not found", 404, {"error": "not found"}) + + lq = AsyncLumenqraphClient(base_url="http://test") + with patch.object(lq, "_request", side_effect=mock_request): + with self.assertRaises(LumenqraphError) as ctx: + await lq.get_interface("UNKNOWN") + self.assertEqual(ctx.exception.status, 404) + + async def test_graphql_raises_on_errors_field(self): + async def mock_request(method, path, query=None, body=None): + return {"errors": [{"message": "field not found"}], "data": None} + + lq = AsyncLumenqraphClient(base_url="http://test") + with patch.object(lq, "_request", side_effect=mock_request): + with self.assertRaises(LumenqraphError): + await lq.graphql("{ bad }") + + +class TestAsyncClientContextManager(unittest.IsolatedAsyncioTestCase): + """Async context manager protocol.""" + + async def test_async_with(self): + async with AsyncLumenqraphClient(base_url="http://test") as lq: + self.assertIsInstance(lq, AsyncLumenqraphClient) + # No exception — aclose is a no-op but must not raise. + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk/python/tests/test_webhook.py b/sdk/python/tests/test_webhook.py index 698d4ec..246e107 100644 --- a/sdk/python/tests/test_webhook.py +++ b/sdk/python/tests/test_webhook.py @@ -1,68 +1,73 @@ """Tests for webhook signature verification.""" +import hmac +import hashlib import unittest + from lumenqraph import verify_webhook_signature +def _sign(body: str, secret: str) -> str: + """Helper: compute the canonical ``sha256=`` signature.""" + digest = hmac.new( + secret.encode("utf-8"), + body.encode("utf-8"), + hashlib.sha256, + ).hexdigest() + return f"sha256={digest}" + + class TestWebhookSignature(unittest.TestCase): """Test webhook signature verification.""" def test_valid_signature(self): - """Test verifying a valid signature.""" + """Valid body + secret → True.""" body = '{"event": "test"}' secret = "test-secret" - # Pre-computed HMAC-SHA256 of body with secret - signature = "9b80d3e1d0b7d7d0b0e0e1e2e3e4e5e6e7e8e9e0e1e2e3e4e5e6e7e8e9e" - - # We'll compute it directly for testing - import hmac - import hashlib - computed_sig = hmac.new( - secret.encode("utf-8"), - body.encode("utf-8"), - hashlib.sha256 - ).hexdigest() - - # Verify with the computed signature - self.assertTrue(verify_webhook_signature(body, computed_sig, secret)) + sig = _sign(body, secret) + self.assertTrue(verify_webhook_signature(body, sig, secret)) def test_invalid_signature(self): - """Test that invalid signature fails.""" + """Tampered hex value → False.""" body = '{"event": "test"}' secret = "test-secret" - invalid_sig = "invalid-signature-here" - - self.assertFalse(verify_webhook_signature(body, invalid_sig, secret)) + self.assertFalse(verify_webhook_signature(body, "sha256=deadbeef", secret)) def test_wrong_secret(self): - """Test that wrong secret fails verification.""" + """Correct format but wrong secret → False.""" body = '{"event": "test"}' - secret = "test-secret" - wrong_secret = "wrong-secret" + sig = _sign(body, "correct-secret") + self.assertFalse(verify_webhook_signature(body, sig, "wrong-secret")) - import hmac - import hashlib - signature = hmac.new( + def test_missing_prefix(self): + """Signature without ``sha256=`` prefix → False.""" + body = '{"event": "test"}' + secret = "test-secret" + bare_hex = hmac.new( secret.encode("utf-8"), body.encode("utf-8"), - hashlib.sha256 + hashlib.sha256, ).hexdigest() - - self.assertFalse(verify_webhook_signature(body, signature, wrong_secret)) + # No "sha256=" prefix → should be rejected + self.assertFalse(verify_webhook_signature(body, bare_hex, secret)) def test_empty_signature(self): - """Test that empty signature fails.""" - body = '{"event": "test"}' - secret = "test-secret" - - self.assertFalse(verify_webhook_signature(body, "", secret)) + """Empty signature header → False.""" + self.assertFalse(verify_webhook_signature('{"event": "test"}', "", "secret")) def test_empty_secret(self): - """Test that empty secret fails.""" + """Empty secret → False (guard against misconfiguration).""" body = '{"event": "test"}' - signature = "some-signature" - - self.assertFalse(verify_webhook_signature(body, signature, "")) + sig = _sign(body, "real-secret") + self.assertFalse(verify_webhook_signature(body, sig, "")) + + def test_body_tampered(self): + """Signature over original body does not verify against modified body.""" + original = '{"amount": "100"}' + tampered = '{"amount": "999"}' + secret = "s3cr3t" + sig = _sign(original, secret) + self.assertFalse(verify_webhook_signature(tampered, sig, secret)) if __name__ == "__main__": diff --git a/sdk/typescript/scripts/check-codegen.mjs b/sdk/typescript/scripts/check-codegen.mjs index aa3001b..fcd123f 100644 --- a/sdk/typescript/scripts/check-codegen.mjs +++ b/sdk/typescript/scripts/check-codegen.mjs @@ -1,7 +1,10 @@ #!/usr/bin/env node /** - * Drift check (#82): re-run openapi-typescript into a temp file, diff it - * against the committed generated/api.d.ts, and exit non-zero if they differ. + * Drift check (#246): Verify that committed TypeScript types match the current openapi.yaml. + * + * This script regenerates types from the current openapi.yaml using openapi-typescript + * and compares the result against the committed generated/api.d.ts. It fails if they differ, + * ensuring hand-edits to the YAML or schema changes never silently diverge from generated types. * * Usage (from sdk/typescript/): * node scripts/check-codegen.mjs @@ -17,34 +20,40 @@ import { randomBytes } from "node:crypto"; const __dirname = dirname(fileURLToPath(import.meta.url)); const root = join(__dirname, ".."); +const openapi = join(root, "..", "..", "openapi.yaml"); const committed = join(root, "generated", "api.d.ts"); const tmp = join(tmpdir(), `lumenqraph-codegen-${randomBytes(6).toString("hex")}.d.ts`); -// Generate fresh into a temp file. +console.log(`ℹ️ Regenerating TypeScript types from ${openapi}...`); + +// Generate fresh types into a temp file from the current openapi.yaml. try { execSync( - `npx openapi-typescript ../../openapi.yaml -o ${tmp}`, + `npx openapi-typescript "${openapi}" -o "${tmp}"`, { cwd: root, stdio: "pipe" }, ); } catch (err) { - console.error("codegen failed:", err.message ?? err); + console.error("❌ Codegen failed:", err.message ?? err); process.exit(1); } -// Compare. +console.log(`ℹ️ Comparing generated types against ${committed}...`); + +// Compare regenerated types with the committed version. const fresh = readFileSync(tmp, "utf8"); const current = existsSync(committed) ? readFileSync(committed, "utf8") : ""; -unlinkSync(tmp); if (fresh !== current) { console.error( - "❌ Generated types are stale!\n" + - " Run `npm run codegen` in sdk/typescript/ and commit the result.\n" + - "\n" + - " The OpenAPI schema (openapi.yaml at the repo root) was updated but the\n" + - " committed generated/api.d.ts was not regenerated.\n", + "❌ Generated types are out of sync with openapi.yaml!\n" + + " The OpenAPI schema at the repo root was updated but the committed\n" + + " generated/api.d.ts does not match the current schema.\n\n" + + " To fix this, run:\n" + + " cd sdk/typescript && npm run codegen && git add generated/api.d.ts && git commit\n", ); + unlinkSync(tmp); process.exit(1); } -console.log("✅ Generated types are up to date."); +unlinkSync(tmp); +console.log("✅ Generated types are in sync with openapi.yaml."); diff --git a/sdk/typescript/src/index.test.ts b/sdk/typescript/src/index.test.ts index 6eb7d88..76f4e04 100644 --- a/sdk/typescript/src/index.test.ts +++ b/sdk/typescript/src/index.test.ts @@ -317,4 +317,54 @@ describe("paginateEvents", () => { ) as { variables: { first: number } }; expect(body.variables.first).toBe(25); }); + + it("aborts the in-flight page request when the consumer breaks early (#279)", async () => { + // Every page reports hasNextPage:true, so the only way iteration ends is the + // consumer bailing out — which must not leave a page fetch running. + const f = mockFetch([ + { status: 200, body: gqlPage(["e1", "e2"], true, "cur-1") }, + { status: 200, body: gqlPage(["e3", "e4"], true, "cur-2") }, + ]); + const c = client(f); + const spy = vi.spyOn(c, "eventsPage"); + + for await (const _ev of c.paginateEvents("C1")) { + break; // stop after the very first event + } + + // The generator's finally cleanup aborts the signal it handed to eventsPage. + const passedSignal = (spy.mock.calls[0]?.[1] as { signal?: AbortSignal }) + .signal; + expect(passedSignal).toBeInstanceOf(AbortSignal); + expect(passedSignal?.aborted).toBe(true); + // And no further page was requested after the early break. + expect(f).toHaveBeenCalledTimes(1); + }); + + it("stops pagination when the caller's AbortSignal fires (#279)", async () => { + const ac = new AbortController(); + const f = mockFetch([ + { status: 200, body: gqlPage(["e1"], true, "cur-1") }, + { status: 200, body: gqlPage(["e2"], true, "cur-2") }, + ]); + const gen = client(f).paginateEvents("C1", { signal: ac.signal }); + + const first = await gen.next(); + expect(first.done).toBe(false); + + ac.abort(); + await expect(gen.next()).rejects.toThrow(/abort/i); + expect(f).toHaveBeenCalledTimes(1); + }); + + it("does not start any request when given an already-aborted signal (#279)", async () => { + const f = mockFetch([ + { status: 200, body: gqlPage(["e1"], true, "cur-1") }, + ]); + const gen = client(f).paginateEvents("C1", { + signal: AbortSignal.abort(), + }); + await expect(gen.next()).rejects.toThrow(/abort/i); + expect(f).not.toHaveBeenCalled(); + }); }); diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 3f40223..f618301 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -98,6 +98,13 @@ export interface DataKeyHistory { versions: { ledger: number; value: Json; captured_at: string }[]; } +export interface ContractsResponse { + data: Contract[]; + has_more: boolean; + /** Pass as `after` on the next call to fetch the next page. `null` when no more pages. */ + next_cursor: string | null; +} + export interface EventsResponse { data: EventRecord[]; has_more: boolean; @@ -123,6 +130,38 @@ export interface CallOptions { sourceAccount?: string; } +export interface Webhook { + id: string; + url: string; + subscriptions: string[]; + active: boolean; + secret: string; + created_at: string; + updated_at: string; +} + +export interface WebhookDelivery { + id: string; + webhook_id: string; + status: "pending" | "success" | "failed"; + status_code?: number; + error?: string; + attempts: number; + created_at: string; + updated_at: string; +} + +export interface CreateWebhookOptions { + url: string; + subscriptions?: string[]; +} + +export interface UpdateWebhookOptions { + url?: string; + subscriptions?: string[]; + active?: boolean; +} + /** A Relay-style page returned by the GraphQL cursor connections. */ export interface Page { nodes: T[]; @@ -237,9 +276,15 @@ export class LumenqraphClient { return this.get("/health", {}, opts.signal); } - /** Contracts the indexer has seen, with per-contract event counts. */ - listContracts(opts: RequestOptions = {}): Promise { - return this.get("/contracts", {}, opts.signal); + /** + * Contracts the indexer has seen, with per-contract event counts. + * Supports cursor-based pagination: pass the `next_cursor` from a previous + * response as `after` to fetch the next page. + */ + listContracts( + opts: { limit?: number; after?: string; signal?: AbortSignal } = {}, + ): Promise { + return this.get("/contracts", { limit: opts.limit, after: opts.after }, opts.signal); } /** A contract's decoded on-chain interface (functions, events, types). */ @@ -286,6 +331,21 @@ export class LumenqraphClient { }, opts.signal); } + /** Fetch a single event by its unique ID. */ + getEvent(eventId: string, opts: RequestOptions = {}): Promise { + return this.get(`/events/${enc(eventId)}`, {}, opts.signal); + } + + /** All indexed events emitted by a transaction, in emission order. */ + getTransactionEvents( + txHash: string, + opts: { limit?: number; signal?: AbortSignal } = {}, + ): Promise<{ tx_hash: string; count: number; data: EventRecord[] }> { + return this.get(`/transactions/${enc(txHash)}/events`, { + limit: opts.limit, + }, opts.signal); + } + /** Materialized SEP-41 transfers, newest first (limit/offset). */ listTransfers( contractId?: string, @@ -320,6 +380,43 @@ export class LumenqraphClient { }, opts.signal); } + // ---- Webhooks ---- + + /** Create a new webhook subscription. Returns the webhook with its secret. */ + createWebhook(opts: CreateWebhookOptions & RequestOptions): Promise { + return this.post("/webhooks", { + url: opts.url, + subscriptions: opts.subscriptions ?? [], + }, opts.signal); + } + + /** List all webhooks for this instance. */ + listWebhooks(opts: RequestOptions = {}): Promise { + return this.get("/webhooks", {}, opts.signal); + } + + /** Delete a webhook by ID. */ + deleteWebhook(id: string, opts: RequestOptions = {}): Promise { + return this.delete(`/webhooks/${enc(id)}`, opts.signal); + } + + /** Update a webhook's URL, subscriptions, or active status. */ + updateWebhook(id: string, opts: UpdateWebhookOptions & RequestOptions): Promise { + return this.post(`/webhooks/${enc(id)}`, { + url: opts.url, + subscriptions: opts.subscriptions, + active: opts.active, + }, opts.signal); + } + + /** List delivery attempts for a webhook. */ + listDeliveries(id: string, opts: { limit?: number; offset?: number; signal?: AbortSignal } = {}): Promise { + return this.get(`/webhooks/${enc(id)}/deliveries`, { + limit: opts.limit, + offset: opts.offset, + }, opts.signal); + } + // ---- GraphQL ---- /** Execute a raw GraphQL query against `/graphql`. */ @@ -379,22 +476,45 @@ export class LumenqraphClient { /** * Async iterator over *all* of a contract's events via GraphQL cursor * pagination — transparently fetching page after page. + * + * Cancellation (#279): pass `signal` to cancel from the caller. Independently, + * breaking out of the `for await` loop early runs this generator's `finally` + * cleanup, which aborts the signal handed to the page fetches — so an + * in-flight `eventsPage` request is cancelled rather than left to run and have + * its result discarded. */ async *paginateEvents( contractId: string, opts: { pageSize?: number; eventName?: string; signal?: AbortSignal } = {}, ): AsyncGenerator { - let after: string | undefined; - for (;;) { - const page = await this.eventsPage(contractId, { - first: opts.pageSize ?? 100, - after, - eventName: opts.eventName, - signal: opts.signal, - }); - for (const node of page.nodes) yield node; - if (!page.hasNextPage || !page.endCursor) return; - after = page.endCursor; + // An internal controller, chained to the caller's signal, is what the page + // fetches actually see. Early termination (a `break` in the consumer's + // `for await`, or a thrown error) triggers the `finally` below, which aborts + // it and tears down any pending request. + const pageAborter = new AbortController(); + const external = opts.signal; + const onExternalAbort = () => pageAborter.abort(external?.reason); + if (external) { + if (external.aborted) pageAborter.abort(external.reason); + else external.addEventListener("abort", onExternalAbort, { once: true }); + } + + try { + let after: string | undefined; + for (;;) { + const page = await this.eventsPage(contractId, { + first: opts.pageSize ?? 100, + after, + eventName: opts.eventName, + signal: pageAborter.signal, + }); + for (const node of page.nodes) yield node; + if (!page.hasNextPage || !page.endCursor) return; + after = page.endCursor; + } + } finally { + external?.removeEventListener("abort", onExternalAbort); + pageAborter.abort(); } } @@ -468,6 +588,12 @@ export class LumenqraphClient { }, signal); } + private async delete(path: string, signal?: AbortSignal): Promise { + return this.request(this.baseUrl + path, { + method: "DELETE", + }, signal); + } + /** * Core fetch wrapper with retry + timeout (#81, #142). * @@ -603,7 +729,7 @@ export async function verifyWebhook( const bodyBytes: ArrayBuffer = typeof rawBody === "string" ? (enc.encode(rawBody).buffer as ArrayBuffer) - : (rawBody.buffer as ArrayBuffer); + : (rawBody.buffer.slice(rawBody.byteOffset, rawBody.byteOffset + rawBody.byteLength) as ArrayBuffer); // Import the secret as an HMAC-SHA-256 key via Web Crypto (Node 18+, browsers). const cryptoKey = await crypto.subtle.importKey(