sqe.toml.example

[coordinator]
flight_sql_port = 50051
trino_http_port = 8080
mode = "hybrid"   # "local", "distributed", or "hybrid"
# debug = false   # Set true for verbose error messages (dev only)
# production_mode = false  # Set true to refuse startup on dev-only settings
# (anonymous/bearer_passthrough auth, disabled rate limiting,
# security.allow_insecure_transport, unauthenticated workers). Env:
# SQE_PRODUCTION_MODE or SQE_COORDINATOR__PRODUCTION_MODE.

# Remote Model Context Protocol endpoint (disabled by default). The MCP bearer
# is validated by the same auth chain as Flight/Trino and forwarded unchanged
# through SQE to Polaris/source storage. The public URL is the OAuth resource
# indicator/audience; configure the bearer-token provider with the same audience.
# [mcp]
# enabled = true  # with port omitted/0, mount on health/web (prometheus_port + 1)
# port = 8081     # optional: isolated listener instead
# resource_uri = "https://sqe.example.com/mcp"
# authorization_servers = ["https://idp.example.com/realms/data"]
# allowed_origins = [] # exact browser origins; native clients send no Origin
# read_scope = "sqe:mcp:read"
# write_scope = "sqe:mcp:write"
# allow_writes = false
# write_roles = ["data_writer"]
# max_result_rows = 1000
# max_response_bytes = 2097152
#
# Deployments with NO identity provider (Glue / S3 Tables / attached warehouse
# reached with engine credentials) have no user tokens to validate. Two modes
# cover that: `static` matches the caller's bearer against configured
# principals, `anonymous` accepts everyone as one identity. Anonymous is
# refused when an IdP provider is configured, under production_mode, and with
# allow_writes = true. See docs deployment/mcp.md.
# [mcp.auth]
# mode = "static"
# [[mcp.auth.principals]]
# user      = "mcp-reader"
# roles     = ["reader"]
# token_env = "SQE_MCP_READ_TOKEN"   # preferred; wins over token_b64
# [[mcp.auth.principals]]
# user      = "mcp-writer"
# roles     = ["data_writer"]
# token_b64 = "..."                  # base64 obfuscates, it does not protect

# Rate limiting (P0 for production). Disabled by default for dev ergonomics.
# Set enabled = true (or SQE_RATE_LIMIT__ENABLED=true) in prod; the
# production_mode validator will refuse startup if disabled.
[rate_limit]
enabled = false
per_user_queries_per_minute = 60
global_queries_per_minute = 1000
auth_attempts_per_minute = 10
metadata_per_user_per_minute = 120

# Memory limit for coordinator query processing (sorts, joins, aggregates).
# Supports: B, KB, MB, GB, TB (e.g. "8GB", "4096MB").
memory_limit = "8GB"
# Spill to disk when memory limit is reached. Recommended: true for production.
spill_to_disk = true
# Directory for spill files. Use fast local SSD. Will be created if it doesn't exist.
spill_dir = "/tmp/sqe-coordinator-spill"
# Compression for spill files: "none", "lz4" (fast, recommended), "zstd" (smaller)
spill_compression = "lz4"

# Optional TLS for Flight SQL listener.
# When cert_file + key_file are set, the server runs with TLS.
# When omitted, the server runs in plaintext.
# [coordinator.tls]
# cert_file = "/etc/sqe/server.crt"
# key_file  = "/etc/sqe/server.key"
# ca_file   = ""  # Optional: PEM CA for mTLS client cert verification

[worker]
coordinator_url = "http://coordinator:50052"
heartbeat_interval_secs = 5
memory_limit = "8GB"
spill_dir = "/tmp/sqe-spill"
# URL the coordinator uses to reach this worker's Flight service, sent in every
# heartbeat. Leave empty to auto-derive a routable address at startup:
#   1. POD_IP env var (Kubernetes downward API)
#   2. HOSTNAME env var, only when it is an IP literal
#   3. first non-loopback, non-link-local local interface address (IPv4 preferred)
# Set explicitly when auto-derivation is wrong (NAT, multi-homed, overlay nets).
# NEVER advertise 0.0.0.0: the coordinator rejects it (every worker would
# collide on one bogus loopback registry entry). Scheme follows TLS:
# https when [coordinator.tls] is set, otherwise http.
# advertise_url = "http://worker-1.svc.cluster.local:50052"
#
# Shared secret authenticating worker <-> coordinator traffic. REQUIRED when
# coordinator_url is set: the worker refuses to boot with an empty secret
# (set worker.allow_unauthenticated = true to waive, dev only). Must match
# coordinator.worker_secret.
worker_secret = "change-me-shared-worker-secret"

[auth]
keycloak_url = "https://auth.local"
realm = "iceberg"
client_id = "sqe-client"
client_secret = ""
token_refresh_buffer_secs = 60
# ssl_verification = false  # ⚠ Only set to false for local dev with self-signed certs

# ── Auth providers (explicit chain, overrides legacy keycloak_url/client_id) ──
# When configured, the auth chain uses these providers in order.
# First provider to accept the credentials wins.

# OIDC password grant — for Trino JDBC / HTTP basic auth logins
[[auth.providers]]
type = "oidc_password"
token_url = "https://auth.local/realms/iceberg/protocol/openid-connect/token"
client_id = "sqe-client"
client_secret = ""
roles_claim = "realm_access.roles"
# groups_claim = "groups"  # JWT claim carrying group membership. Required for
#                          # Ranger policies bound only to a group (no users/roles).

# Bearer token (JWT) — for pre-authenticated clients (data platform, CI/CD, Airflow)
# IMPORTANT: issuer must match KC_HOSTNAME exactly. If KC_HOSTNAME includes a scheme
# (e.g. "http://auth.local"), the issuer here must use the same scheme.
[[auth.providers]]
type = "bearer_token"
jwks_url = "https://auth.local/realms/iceberg/protocol/openid-connect/certs"
issuer = "https://auth.local/realms/iceberg"
# audience = "sqe"  # ⚠ Recommended: restrict accepted tokens to this audience claim.
                     # Without this, tokens issued for ANY service by this IdP are accepted.
user_claim = "preferred_username"
roles_claim = "realm_access.roles"
# groups_claim = "groups"  # JWT claim carrying group membership. Required for
#                          # Ranger policies bound only to a group (no users/roles).

# ── Interactive OIDC flows (Trino SSO + CLI device code) ────────
# Uncomment to enable browser-based SSO for Trino JDBC and device code for CLI.
#
# The Authorization Code + PKCE flow is used for Trino external authentication:
#   1. Trino JDBC sends query without credentials
#   2. SQE returns 401 with WWW-Authenticate: Bearer x_redirect_server="...", x_token_server="..."
#   3. JDBC driver opens browser for IdP login
#   4. User authenticates, SQE receives callback, JDBC polls for token
#
# The Device Code flow (RFC 8628) is used for CLI / headless environments:
#   user runs `sqe auth login`, gets a URL + code, authenticates in browser.
#
# [auth.external]
# issuer = "https://idp.example.com/realms/sqe"
# client_id = "sqe"
# client_secret = "your-client-secret"           # Optional: omit for public clients
# redirect_uri = "https://sqe.example.com/oauth2/callback"
# scopes = ["openid", "profile"]
# challenge_timeout_secs = 900                    # How long auth sessions live (default: 900s)
# accept_invalid_certs = false                    # Set true for dev with self-signed certs
#
# # Optional: override OIDC discovery endpoints (auto-discovered from issuer by default)
# # authorization_endpoint = "https://idp.example.com/realms/sqe/protocol/openid-connect/auth"
# # token_endpoint = "https://idp.example.com/realms/sqe/protocol/openid-connect/token"
# # device_authorization_endpoint = "https://idp.example.com/realms/sqe/protocol/openid-connect/auth/device"
#
# # Device code flow — separate client_id allows different scopes (e.g. offline_access)
# [auth.external.device]
# client_id = "sqe-cli"
# scopes = ["openid", "profile", "offline_access"]

[catalog]
polaris_url = "http://polaris:8181/api/catalog"
warehouse = "iceberg"
metadata_cache_ttl_secs = 30
# Deadlines for every HTTP call to the catalog (SQE's own HEAD/GET path and
# iceberg-rust's requests share one client). A silent catalog fails the call
# with CATALOG_UNAVAILABLE and trips the circuit breaker instead of stalling
# planning. Must be >= 1. The client is process-wide: with several
# [catalogs.*] entries the first one loaded sets the values.
# request_timeout_secs = 30
# connect_timeout_secs = 5
default_table_format_version = 2  # 2 (default) or 3 for Iceberg v3 tables
# trust_sort_order = false  # default: only partition columns declared as pre-sorted (safe)
# trust_sort_order = true   # trust ALL Iceberg sort order columns (faster but risky for mixed-writer tables)
# Hide namespace NAMES the caller has no grants in from SHOW SCHEMAS,
# information_schema.schemata, and Flight SQL GetDbSchemas. REST/Polaris
# backend only; probes each listed namespace once per session with the
# caller's bearer (LOAD_NAMESPACE_METADATA) and drops names on 403.
# Fails open on any other probe error. Set false to restore unfiltered
# listings.
# namespace_visibility_filter = true
#
# When true, Iceberg FileIO never receives the configured [storage]
# access/secret keys. Reads then need Polaris-vended STS (or remote
# signing). Default false for single-tenant / local stacks.
# production_mode requires this on every REST catalog (issue #395).
# require_vended_credentials = false

[storage]
s3_endpoint = "http://s3:9000"
s3_region = "us-east-1"
s3_access_key = "<your-s3-access-key>"
s3_secret_key = "<your-s3-secret-key>"
s3_path_style = true

# ── File-TVF security (read_parquet / read_csv / read_json / read_delta) ──
# All defaults fail closed.
#
# [storage.tvf]
# Local filesystem paths in TVF calls (read_csv('/etc/passwd')). Default
# false; leave false on any shared deployment.
# allow_local_paths = false
#
# Arbitrary http(s):// fetches from TVF calls. Default false (IMDS lives at
# http://169.254.169.254). When false, only allowed_http_hosts are reachable.
# allow_http = false
# allowed_http_hosts = ["huggingface.co"]
#
# Object-store URL prefixes (s3:// s3a:// abfss:// abfs:// azure:// az://
# gs:// gcs://) that TVFs may read using the ENGINE's storage credentials
# above. Default EMPTY = all such reads are DENIED: without this gate any
# authenticated user could read_csv('s3://<any-bucket>/<any-key>') with the
# engine's static S3 key, bypassing catalog authorization entirely. Prefixes
# match on path-segment boundaries; the literal `{user}` expands to the
# authenticated username. TVF calls that carry complete inline credentials
# (access_key + secret_key, an Azure key / SAS token, or an inline GCS
# service-account key) bypass this list — the engine's key is not used.
# allowed_object_store_prefixes = [
#   "s3://data-platform-staging/_table-load-staging/",
#   "s3://notebook-scratch/{user}/",
# ]
#
# JWT roles (case-insensitive) allowed to read ANY object-store path with
# the engine's credentials. Default empty = no override.
# object_store_admin_roles = ["sqe-storage-admin"]
#
# Whether inline TVF credentials bypass the prefix allowlist above (SEC-04).
# Default true: a call carrying its own access_key/secret_key (or Azure/GCS
# equivalent) reads with those creds, so the object store enforces access and
# the allowlist is skipped. Set false on a multi-tenant coordinator that does
# NOT want users bringing their own object-store credentials; inline-cred reads
# then still have to pass allowed_object_store_prefixes / object_store_admin_roles.
# allow_inline_credentials = false

# ── Hive-style external tables (CREATE EXTERNAL TABLE, Athena/Glue DDL) ──
# Read-only in cycle 1: CSV, line-delimited JSON, and Parquet, against a
# sidecar JSON manifest under an attached warehouse root, no metastore.
# See docs/site/book/src/reference/hive-external-tables.md.
#
# [hive]
# Manifest caps enforced on every root, at CREATE EXTERNAL TABLE and on
# every load: the manifest is untrusted input, planted by anyone who can
# write to the warehouse prefix. These four match
# hive::manifest::ManifestLimits::default() exactly; a mismatch here would
# be a config surface that silently does nothing.
# manifest_max_bytes = 1048576          # 1 MiB
# manifest_max_columns = 4096
# manifest_max_partition_keys = 32
# manifest_max_partition_index_entries = 100000
#
# Warehouse roots attached before the first query, the config equivalent
# of ATTACH '<location>' AS <name> (TYPE hive_external). Repeatable.
# [[hive.roots]]
# name     = "lake"
# location = "s3://raw-lake/warehouse/"
# secret   = "partner"  # optional; a table's own TBLPROPERTIES 'sqe.secret' wins

# ── Wire-protocol security ────────────────────────────────────
[security]
# trusted_proxies = ["10.0.0.1"]  # IPs allowed to set x-forwarded-for for audit IPs
#
# allow_insecure_transport: opt out of the TLS-on-the-wire requirement.
# When distributed mode targets a NON-loopback peer (coordinator.worker_urls
# or worker.coordinator_url pointing off-box) and [coordinator.tls] is not
# configured, the engine refuses to start. User bearer tokens, the worker
# secret, and vended S3 credentials would otherwise travel in cleartext.
# Loopback-only setups (127.0.0.1 / ::1 / localhost) stay usable without TLS.
#
# This example talks to a non-loopback "coordinator" host over plaintext http,
# so the opt-in is enabled here for dev. Remove it and configure
# [coordinator.tls] before exposing the engine on an untrusted network.
allow_insecure_transport = true  # dev only -- enable TLS for production
#
# allow_shared_service_identity (SEC-03): a config-held `client_credentials`
# grant hands ONE server-baked service token to every connection, so Polaris/S3
# see a single identity for all users and per-user authorization at the data
# layer is lost. Two config paths carry this risk and are gated together:
#   - a `client_credentials` provider in [[auth.providers]] (user auth), and
#   - a `[catalog.auth]` / `[catalogs.<name>.auth]` block with
#     method = client_credentials (catalog auth reused across the session).
# In production_mode the engine refuses to start with either unless this is set
# true. For multi-tenant access use `oidc_password` (per-user) or
# `client_credentials_passthrough` (per-connection) for user auth and the
# default SessionBearer for catalog auth. Set true only for a deliberate
# single-service deployment where every caller legitimately shares one identity.
# allow_shared_service_identity = false
#
# allow_unkeyed_hash_masks (issue #402): Ranger MASK_HASH falls back to
# unsalted SHA-256 when policy.mask_key is empty (rainbow-tableable on
# SSN/phone/small enums). In production_mode the engine refuses to start
# with engine = ranger and an empty mask_key unless this is set true.
# Set policy.mask_key (or SQE_POLICY__MASK_KEY) instead.
# allow_unkeyed_hash_masks = false
#
# allow_unprojected_tags (issue #397): SET TAG writes Iceberg
# `sqe.column-tags` (SQE-only). Spark/Kyuubi masks from Ranger's tag store.
# With engine = ranger and project-tags off, SQE masks and Spark returns
# raw. Mixed SQE+Spark MUST set policy.ranger.project-tags = true.
# production_mode refuses Ranger with project-tags off unless this is set
# true (SQE-only Ranger deploys that never talk to Spark).
# allow_unprojected_tags = false

[policy]
engine = "passthrough"
# Required in production_mode when engine = "ranger" (unless
# security.allow_unkeyed_hash_masks = true). Upgrades MASK_HASH to HMAC.
# mask_key = "set-via-SQE_POLICY__MASK_KEY"
#
# [policy.ranger]
# Mixed SQE+Spark MUST set project-tags = true so Spark sees the same
# column tags SQE masks. Default is false (SQE-only Ranger deploys).
# project-tags = true

[session]
idle_timeout_secs = 900       # 15 minutes — sessions idle longer than this are expired
absolute_timeout_secs = 28800  # 8 hours — sessions older than this are expired regardless of activity
# Optional CREATE SECRET snapshot (plaintext JSON, chmod 0600). Empty = memory
# only (lost on restart). ATTACH mounts are still process-local. #409
# secrets_path = "/var/lib/sqe/secrets.json"

[metrics]
prometheus_port = 9090
# OTLP gRPC endpoint for traces, metrics, and logs (e.g. "http://otel-collector:4317")
# Leave empty to disable OpenTelemetry export (structured JSON logs only)
otlp_endpoint = ""
# Trace-only OTLP gRPC endpoint. Prefer this when metrics are scraped from
# /metrics and logs are collected from stdout.
traces_otlp_endpoint = ""
# Security audit log (one JSONL entry per statement: who ran what, when, against
# which tables). Empty disables it (the logger is a no-op). The default is empty
# here because an unwritable path is fatal at startup, and a single example file
# cannot know your durable location; leaving it off keeps a plain `cargo run`
# and the quickstarts working.
#
# DURABILITY: the path MUST live on storage that survives process/host restart.
# Records appended to an ephemeral location (a container emptyDir, a tmpfs, /tmp)
# vanish on restart and the audit trail is lost. For a sovereign, fine-grained
# security engine that is a compliance gap, so point this at a persistent mount,
# e.g. a dedicated volume:
#   audit_log_path = "/var/log/sqe/audit/audit.jsonl"
# Under Helm this is wired automatically: set audit.enabled=true (default) plus
# audit.persistence.enabled=true to back it with a PVC. See ISSUE-248.
audit_log_path = ""

# OpenTelemetry trace sampling (0.0–1.0). Default when omitted: 0.01 (1% of queries).
# Set to 1.0 to trace every query (expensive at high QPS). Set to 0.0 to disable tracing.
# trace_sample_rate = 0.01
# Environment override: SQE_METRICS__TRACE_SAMPLE_RATE=1.0

# OTLP audit export: tails the OCSF spool and ships records to a SIEM collector.
# Off by default. Requires audit_log_path (or metrics.audit format = "ocsf"/"both").
# Production tuning and phase-by-phase guidance: docs/production.md
[metrics.audit_export]
enabled = false
target = "otlp"               # only "otlp" is supported today; "kafka" is reserved
otlp_endpoint = ""            # empty falls back to metrics.otlp_endpoint above
spool_path = ""               # empty -> <audit_log_path>.ocsf.spool.jsonl
batch_max = 512               # max OCSF records per OTLP export batch
flush_interval_ms = 2000      # shipper poll interval in milliseconds
max_spool_bytes = 1073741824  # 1 GiB; WARN when spool exceeds this (queries never block)
start_at = "now"              # "now" (skip backlog on first run) | "beginning" (replay all)

# OpenLineage 2-0-2 lineage events for DML/DDL. Off by default; zero hot-path cost when
# disabled. When enabled, at least one sink (file_path or http_endpoint) is required.
# Full reference: docs/site/book/src/operations/openlineage.md
# Production tuning: docs/production.md
[metrics.openlineage]
enabled = false
job_namespace = "sqe"         # per-environment namespace in lineage UI
# producer = ""               # optional producer URI override (default: sqe binary version)
emit_selects = false          # set true only when read lineage is required
file_path = ""                # append-only JSONL sink (e.g. "/var/log/sqe/lineage.jsonl")
http_endpoint = ""            # OL collector URL (e.g. "https://marquez.example.com/api/v1/lineage")
auth_mode = "none"            # "none" | "bearer" | "user_token"
api_key = ""                  # required when auth_mode = "bearer"
http_timeout_ms = 5000
http_retry_attempts = 1
spool_path = ""               # disk spool for HTTP collector outages (recommended with http_endpoint)
spool_max_bytes = 104857600   # 100 MiB default
replay_interval_secs = 30
channel_capacity = 10000

# ── Query execution ───────────────────────────────────────────
[query]
# Maximum estimated build-side size for hash join before falling back to
# SortMergeJoin. SortMergeJoin spills to disk via external sort, preventing
# OOM on large joins. Default: "2GB". Set to "0" to always use hash join.
# Supports: B, KB, MB, GB, TB.
hash_join_memory_threshold = "2GB"
# Maximum SQL statement size in bytes, checked before parse (issue #403).
# Default: 1048576 (1 MiB). Set to 0 for unlimited; production_mode refuses
# 0 so a production coordinator cannot accept arbitrarily large statements.
# max_sql_bytes = 1048576
# Cap concurrent CTAS/INSERT with a top-level ORDER BY (issue #408). Extra
# sorted writes are rejected with ResourceExhausted, not queued. Default 2.
# A dbt or bench run that used to fire 4 parallel sorted loads will fail 2
# of them at this default; raise the cap or drop ORDER BY on the write.
# max_concurrent_sorted_writes = 2

# ── Query result cache ────────────────────────────────────────
[query_cache]
# Enable/disable the query result cache.
enabled = true
# Maximum total cache memory in MB.
max_memory_mb = 256
# Maximum size of a single cached result in MB. Results larger than this
# are not cached.
max_entry_mb = 5
# Time-to-live in seconds. Cached results expire after this duration.
ttl_secs = 300

# ── Query history ─────────────────────────────────────────────
[query_history]
# Maximum number of query records kept in memory (viewable via
# SELECT * FROM system.runtime.queries).
max_entries = 10000
# Time-to-live in seconds for query history records.
ttl_secs = 1800

# ── Production example (copy/adapt for real deploys) ─────────────────
# Uncomment/adjust for production. Combine with production_mode = true
# and the P0 checklist in docs/internal/audit/2026-07-10-sqe-full-audit.md.
#
# [coordinator]
# production_mode = true
# memory_limit = "64GB"   # honest fraction of host RAM
# spill_to_disk = true
# spill_dir = "/fast/ssd/sqe-spill"
#
# [rate_limit]
# enabled = true
# per_user_queries_per_minute = 120
# global_queries_per_minute = 5000
#
# [coordinator.tls]
# cert_file = "/etc/sqe/tls.crt"
# key_file  = "/etc/sqe/tls.key"
#
# [worker]
# worker_secret = "long-random-secret-here"
# advertise_url = "https://worker-1:50052"
#
# [metrics]
# otlp_endpoint = "http://otel:4317"
# trace_sample_rate = 0.05
# audit_log_path = "/var/log/sqe/audit/audit.jsonl"
#
# [metrics.audit_export]
# enabled = true
# spool_path = "/var/log/sqe/audit/audit.ocsf.spool.jsonl"
#
# [metrics.openlineage]
# enabled = true
# http_endpoint = "https://marquez/api/v1/lineage"
# Memory / spill / jemalloc notes (from full audit P1/M section)
# Use honest memory_limit (e.g. 50-70% RAM). spill_to_disk=true + fast NVMe.
# For jemalloc (M6): consider in Docker/Helm for background purge to reduce
# RSS parking post-query. Test via benchmark rig.
# See audit for full M1-M8 matrix and decision.