config.yml

# Model Configuration
# This file defines all model settings for the RAG system.
# API keys are stored separately in secrets/.env

# ============================================================================
# Model Definitions - Define all available models here
# ============================================================================
models:
  inference:
    # OpenAI
    # Cheapest current-generation model that still follows the grounding and
    # citation instructions in `prompts.context` ($0.20/$1.20 per 1M tokens).
    # gpt-5-nano ($0.05/$0.40) is cheaper but drops citations under load.
    gpt56-luna:
      provider: openai
      model: gpt-5.6-luna
      base_url: https://api.openai.com/v1
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    gpt5-mini:
      provider: openai
      model: gpt-5-mini
      base_url: https://api.openai.com/v1
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    # Anthropic models (cloud API)
    # Cheapest Anthropic model ($1/$5 per 1M tokens). 
    claude-haiku:
      provider: anthropic
      model: claude-haiku-4-5
      base_url: https://api.anthropic.com
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    claude-sonnet:
      provider: anthropic
      model: claude-sonnet-5
      base_url: https://api.anthropic.com
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    claude-opus:
      provider: anthropic
      model: claude-opus-5
      base_url: https://api.anthropic.com
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    # Google, DeepSeek, and Moonshot are not currently supported — no compose file
    # declares their API keys as Docker secrets, so selecting them passes YAML
    # validation and then fails at boot. Removed; example of what a future cloud
    # provider entry looks like (do not uncomment without also adding: the key field
    # to Settings in BOTH services/rag_server/app/settings.py and
    # services/evals/infrastructure/settings.py, a validator + dispatch entry in
    # infrastructure/llm/validation.py, an import mapping in infrastructure/llm/factory.py,
    # a Provider enum value in infrastructure/llm/config.py, a Docker secret declaration
    # in the compose files, and a cost-table entry):
    #
    # some-provider-model:
    #   provider: some-provider
    #   model: some-model-name
    #   base_url: https://api.some-provider.example/v1
    #   timeout: 120
    #   requires_api_key: true
    #   execution_boundary: third_party

    # Mode B only: an OpenAI-compatible vLLM endpoint on the private L40S in
    # the AWS VPC. `just llm-up` replaces the explicit inactive placeholder
    # with its private address; no laptop-to-AWS route exists or is supported.
    qwen35-9b:
      provider: vllm
      model: Qwen/Qwen3.5-9B
      base_url: http://unconfigured-private-vllm.invalid/v1
      timeout: 120
      execution_boundary: customer_managed

  embedding:
    # NOTE: Switching the active embedding model invalidates every stored vector —
    # the schema must be re-created and all documents re-ingested.

    # Self-hosted TEI (HuggingFace Text Embeddings Inference), 1024-dim.
    # query_instruction is Qwen3's documented asymmetric query format; it is
    # applied ONLY to queries, never to documents — text_instruction is
    # deliberately empty. Prefixing documents would silently degrade retrieval.
    qwen3-embed:
      provider: tei
      model: Qwen/Qwen3-Embedding-0.6B
      base_url: http://tei:80
      embed_batch_size: 32
      timeout: 60
      query_instruction: "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:"
      text_instruction: ""
      execution_boundary: customer_managed

    # OpenAI embeddings (cloud API)
    openai-ada:
      provider: openai
      model: text-embedding-ada-002
      base_url: https://api.openai.com/v1
      requires_api_key: true
      execution_boundary: third_party

    openai-3-small:
      provider: openai
      model: text-embedding-3-small
      base_url: https://api.openai.com/v1
      requires_api_key: true
      embed_batch_size: 100
      execution_boundary: third_party

    openai-3-large:
      provider: openai
      model: text-embedding-3-large
      base_url: https://api.openai.com/v1
      requires_api_key: true
      execution_boundary: third_party

  # Rule of thumb: use a grader at least as capable as your answer (inference) model when you 
  # care about subtle errors or faithfulness; otherwise use a cheaper mini/nano model for scale.
  eval:
    # Anthropic models for evaluation (cloud API)
    claude-sonnet:
      provider: anthropic
      model: claude-sonnet-5
      base_url: https://api.anthropic.com
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    claude-opus:
      provider: anthropic
      model: claude-opus-4-5-20251101
      base_url: https://api.anthropic.com
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    # OpenAI models for evaluation (cloud API)
    gpt5-mini:
      provider: openai
      model: gpt-5-mini
      base_url: https://api.openai.com/v1
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    gpt5-2:
      provider: openai
      model: gpt-5.2
      base_url: https://api.openai.com/v1
      timeout: 120
      requires_api_key: true
      execution_boundary: third_party

    # A self-hosted judge behind the network boundary needs no API key; its
    # execution_boundary is what makes it acceptable for a confidential corpus,
    # not its provider string. Start it with `just judge-up` (CUDA host only —
    # the service is profile-gated so a bare `just up` skips it), then point
    # active.eval here to evaluate your own documents. Override the model with
    # JUDGE_MODEL if 32B at 4-bit does not fit the GPU.
    qwen-vllm-judge:
      provider: vllm
      model: Qwen/Qwen3-32B-AWQ
      base_url: http://vllm:8000/v1
      timeout: 120
      execution_boundary: customer_managed

    # Mode B only: hosted beside qwen35-9b on the private L40S. The official
    # FP8 checkpoint is used directly; `just llm-up` writes its VPC address.
    qwen38-27b-judge:
      provider: vllm
      model: Qwen/Qwen3.8-27B-FP8
      base_url: http://unconfigured-private-vllm.invalid/v1
      timeout: 120
      execution_boundary: customer_managed

  reranker:
    # top_n: number of nodes the reranker returns. Authoritative when set; if
    # omitted, falls back to max(5, retrieval.top_k // 2).
    # Lightweight reranker (default)
    minilm-l6:
      model: cross-encoder/ms-marco-MiniLM-L-6-v2
      top_n: 5

    # More powerful rerankers
    bge-reranker-large:
      model: BAAI/bge-reranker-large
      top_n: 5

    bge-reranker-base:
      model: BAAI/bge-reranker-base
      top_n: 5

# ============================================================================
# Active Model Selection - Specify which models to use
# ============================================================================
active:
  inference: gpt5-mini
  embedding: qwen3-embed
  eval: gpt5-2
  reranker: minilm-l6

# ============================================================================
# Evaluation Settings (non-model-specific)
# ============================================================================
eval:
  # Citation scope for evaluation:
  # - retrieved: treat all retrieved chunks as citations
  # - explicit: only use explicitly cited chunks (if provided by the server)
  #
  # 'explicit' is the shipped default: under 'retrieved' the LLM is never asked
  # to emit inline markers, so citation_entailment and claim_citation_support
  # are always undefined and uncited_claim_rate is a constant 1.0 no matter what
  # the answer says. Those metrics only mean anything once the model is
  # actually producing citations to check.
  citation_scope: explicit

  # Citation format for explicit citations
  # - numeric: uses [1], [2] style references mapped to source order
  citation_format: numeric

  # Phrases used to detect that an answer was a refusal rather than a real answer.
  # Matched case-insensitively as SUBSTRINGS, so keep them as short fragments —
  # a full sentence with terminal punctuation only matches that exact wording and
  # will miss paraphrases ("I don't have enough information to answer that.").
  # Narrowing this list makes the model look worse at abstaining without anything
  # about the model having changed, so treat edits here as a scoring change.
  abstention_phrases:
    - "i don't have enough information"
    - "i do not have enough information"
    - "cannot answer"
    - "can't answer"
    - "unable to answer"
    - "not enough information"
    - "insufficient information"
    - "no information available"
    - "don't know"
    - "do not know"
    - "cannot be determined"
    - "can't be determined"
    - "not mentioned"
    - "not found"
    - "no relevant information"

  # Weighted-score configuration. These decide what the single headline number
  # rewards, so treat any edit here as a scoring change: run comparisons across a
  # change to this block are not comparable.
  scoring:
    # Objective weights. Objectives with no data in a run are dropped and their
    # weight redistributed, so these are relative, not required to sum to 1.
    weights:
      accuracy: 0.30        # answer correctness + abstention
      faithfulness: 0.20    # grounding in the retrieved context
      citation: 0.20        # citation precision / recall / section accuracy
      # Claim-level grounding (claim_groundedness, citation_entailment,
      # claim_citation_support, uncited_claim_rate). 0.0 means the metrics are
      # reported but do not move the headline score — raising it changes what the
      # headline rewards, so runs before and after the change are not comparable.
      # The group runs by default, but remains informational at this weight.
      groundedness: 0.0
      retrieval: 0.15       # recall, precision, MRR, nDCG
      cost: 0.10            # cost per query
      latency: 0.05         # P50 latency

    # Latency normalization: 0 ms scores 1.0, the threshold and above scores 0.0.
    # End-to-end pays retrieval and generation; generation-tier runs skip retrieval,
    # so they are held to a tighter bar. Lower these for a latency-sensitive
    # deployment — leaving them at a profile you do not run to gives you a headline
    # number that does not reflect your constraints.
    latency_threshold_ms_generation: 5000
    latency_threshold_ms_end_to_end: 30000

    # Cost normalization: $0/query scores 1.0, this and above scores 0.0 (USD).
    max_cost_per_query_usd: 0.10

# ============================================================================
# Reranker Settings (non-model-specific)
# ============================================================================
reranker:
  enabled: true

# ============================================================================
# Retrieval Settings
# ============================================================================
# ============================================================================
# Chunking - how documents are split before embedding
# ============================================================================
# Applies to the SentenceSplitter path (.txt/.md). Complex documents go through
# Docling, which splits on document structure and has no size/overlap knob —
# `GET /metrics/retrieval` reports `chunker` so a run records which path it
# actually took, and reports null sizes for the Docling path rather than these
# numbers, which that path never used.
chunking:
  chunk_size: 500
  chunk_overlap: 50

retrieval:
  top_k: 10
  enable_hybrid_search: true    # BM25 + Vector search with RRF fusion
  rrf_k: 60                      # Reciprocal Rank Fusion parameter
  enable_contextual_retrieval: false   # Anthropic contextual retrieval (much slower doc processing)
  contextual_concurrency: 8            # Max concurrent LLM calls during contextual prefix generation
  # Max concurrent embedding batches during ingestion. This exists for the burst
  # GPU embedder (`just embed-up`), where TEI can coalesce concurrent requests
  # into bigger forward passes. Against the always-on CPU `tei` container it buys
  # nothing — measured 1.01x on 160 chunks, because that container is
  # compute-bound, not round-trip-bound. Raising this will not speed up laptop
  # ingestion; it will just queue more work inside TEI.
  embed_concurrency: 8

# ============================================================================
# Vector Store (pgvector + pgvectorscale StreamingDiskANN, in Postgres)
# ============================================================================
vector_store:
  dimension: 1024                # MUST match the active embedding model's output
                                 # dimension. Changing it requires re-creating the
                                 # schema and re-ingesting every document.

# ============================================================================
# Database Connection Pooling
# ============================================================================
database:
  # PostgreSQL server-side max_connections is set directly in docker-compose.yml
  # (postgres service command), not here — it was never read by the application.

  # Application-side connection pool (per service: rag-server, task-worker)
  # Total connections per service = pool_size + max_overflow
  pool_size: 10         # Persistent connections kept open
  max_overflow: 20      # Additional connections allowed under burst load
  pool_pre_ping: true   # Verify connections before use (prevents stale connections)
  pool_recycle: 3600    # Recycle connections after 1 hour (seconds)

# ============================================================================
# Chat Memory Caches (in-process, per rag-server worker)
# ============================================================================
# Both caches hold cleartext user messages in RAM. Bounded by an idle TTL and an
# LRU size cap so a long-lived process does not accumulate every session it ever
# served. Evicted entries are dropped, not persisted.
chat_memory:
  # Cache in front of the PostgreSQL chat store — eviction costs one reload.
  persistent:
    max_sessions: 500
    ttl_seconds: 3600

  # Temporary sessions have no DB copy, so eviction ends the conversation.
  # Shorter on purpose: this is the only bound on cleartext temporary history.
  temporary:
    max_sessions: 200
    ttl_seconds: 1800

# ============================================================================
# Prompts - RAG pipeline prompts
# ============================================================================
prompts:
  # System-level instructions for LLM behavior
  system: |
    You are a professional assistant providing accurate answers based on document context.
    Be direct and concise. Avoid conversational fillers like 'Let me explain', 'Okay', 'Well', or 'Sure'.
    Start responses immediately with the answer.
    Use bullet points for lists when appropriate.

  # Context instructions for using retrieved documents
  # Placeholders: {context_str}, {chat_history}, {citation_instructions}
  context: |
    Context from retrieved documents:
    {context_str}

    Instructions:
    - Answer using ONLY the context provided above
    - If the context does not contain sufficient information, respond: "I don't have enough information to answer this question."
    - Never use prior knowledge or make assumptions beyond what is explicitly stated
    - Be specific and cite details from the context when relevant
    - Use citations consistently when referencing facts{citation_instructions}
    - Previous conversation context is available for reference

    Provide a direct, accurate answer based on the context:

  # Citation instructions (appended to context prompt when citations enabled)
  citation_instructions:
    numeric: |2

      - Add numeric citations in square brackets like [1], [2] that map to the order of context chunks provided above.

  # Question condensation prompt (null = use LlamaIndex default)
  # Default works well: "Given a conversation (between Human and Assistant) and a follow up
  # message from Human, rewrite the message to be a standalone question that captures all relevant context."
  condense: null

  # Contextual prefix generation for chunks (Anthropic method)
  # Placeholders: {document_name}, {document_type}, {chunk_preview}
  contextual_prefix: |
    Document: {document_name} ({document_type})

    Chunk content:
    {chunk_preview}

    Provide a concise 1-2 sentence context for this chunk, explaining what document it's from and what topic it discusses.
    Format: "This section from [document/topic] discusses [specific topic/concept]."

    Context (1-2 sentences only):

# ============================================================================
# Data Policy - where corpus content is allowed to be processed
# ============================================================================
# Deliberately independent of `pii`: content can be commercially confidential
# without containing a single PII entity, and masking is not the control for it.
# Nothing in the eval path is masked at all — judge prompts embed retrieved
# chunks and generated answers verbatim — so this block, not `pii.enabled`, is
# what decides whether a judge endpoint may see the corpus.
#
# `execution_boundary` on each model definition above says where that endpoint
# actually runs. It is never inferred from `provider`: an OpenAI-compatible
# transport can point at a vLLM container we run or at api.openai.com. A model
# that declares no boundary is unknown, and unknown fails closed.
data_policy:
  # Default true. An operator who has said nothing has not said "public".
  corpus_confidential: true

  # Allow-list. Anything not named here is refused for a confidential corpus.
  #   customer_managed - a host/VPC we run (local Docker, our EC2, our K8s)
  #   aws_managed      - Bedrock/SageMaker, inside the customer's AWS boundary
  #   third_party      - OpenAI, Anthropic, any vendor-hosted API
  allowed_judge_boundaries:
    - customer_managed
    - aws_managed

  # The escape hatch, per dataset rather than per deployment. These are public
  # HuggingFace benchmarks, so a judge that sees their questions and gold
  # passages learns nothing about your corpus. `golden` is deliberately absent:
  # it is authored from your own documents, so a golden run is always gated.
  # The gate reads this list per run — a mixed run is as confidential as its
  # most confidential dataset.
  public_datasets:
    - ragbench
    - qasper
    - squad_v2
    - hotpotqa
    - msmarco

  # A public dataset is not sufficient in the END_TO_END tier. There the eval
  # queries the live rag-server index, which returns whatever is in it — your
  # documents included — so the judge sees corpus content regardless of which
  # dataset asked the question. Set true ONLY when the index this eval runs
  # against holds nothing but the eval's own uploaded documents (an ephemeral
  # benchmark stack, a throwaway database). The shipped `just test-eval` runs
  # against the persistent index, so it is false here and that run is gated.
  #
  # The evals service also honours EVAL_INDEX_IS_ISOLATED=true as an override,
  # for ephemeral compose stacks that cannot edit this file. It logs loudly.
  eval_index_is_isolated: false

# ============================================================================
# PII Masking - opt-in cloud generation tier only (Task 2.3)
# ============================================================================
# Reversible PII masking for every path that sends text to a cloud LLM
# provider (openai/anthropic): the generation path (user query, retrieved
# context, chat history, session-title generation) and contextual enrichment
# during ingestion (document name + chunk preview are masked before the LLM
# call; the generated prefix is unmasked before local storage/embedding).
# Embeddings and the reranker are never masked and stay local/VM-side by
# product decision — see validate_privacy_posture() in models_config.py,
# which refuses to boot if pii.enabled is true with a non-local embedding
# provider.
#
# Masked text is pseudonymized, not anonymized: it reduces exposure to the
# cloud provider but is not a compliance guarantee. For genuinely sensitive
# corpora, use the self-hosted TEI embedder (models.embedding.qwen3-embed)
# together with a self-hosted vllm inference model instead of a cloud provider.
pii:
  enabled: false  # Master toggle

  # Entity types to detect and mask. Full list: https://microsoft.github.io/presidio/supported_entities/
  entities:
    - PERSON
    - EMAIL_ADDRESS
    - PHONE_NUMBER
    - CREDIT_CARD
    - US_SSN
    - IBAN_CODE
    - IP_ADDRESS

  # Distinctive bracket tokens, e.g. [[[PERSON_0]]] — the only masking strategy supported.
  token_format: "[[[{entity_type}_{index}]]]"

  # Score threshold for PII detection (0.0-1.0)
  score_threshold: 0.5

  # Language + spaCy NLP model for Presidio's analyzer
  language: en
  spacy_model: en_core_web_md  # en_core_web_lg is more accurate but ~15x larger on disk

  # Optional second opinion on NER-shaped entities (names, phones, addresses),
  # registered alongside spaCy and the regex recognizers rather than replacing
  # them. spaCy is the weak link on PERSON recall; GLiNER scores better and holds
  # up across domains, at ~10x the CPU cost per call (~160ms vs ~15ms).
  # Requires the optional package: `uv sync --extra gliner`.
  gliner:
    enabled: false
    model_name: urchade/gliner_multi_pii-v1  # Apache-2.0, ~200MB on first use
    threshold: 0.4          # GLiNER's own confidence threshold
    map_location: null      # null = auto-detect GPU; "cpu" to force CPU
    load_onnx_model: false  # true for CPUs without AVX2 (needs onnxruntime)

  # Validate that tokens the LLM was given survive unaltered in its response;
  # attempt fuzzy recovery (bracket/case/separator repair) if not
  validation:
    enabled: true

  # Scan the unmasked response for verbatim PII leaks (output guardrail)
  output_guardrails:
    enabled: true
    block_on_detection: false  # if true, raise instead of returning the response

  # Bounds on the in-memory session token mapping (holds cleartext PII, so it is
  # evicted on idle as well as on size). Losing a mapping is safe — the next turn
  # re-masks from persisted history; only token numbering changes.
  session_mapping:
    max_sessions: 500
    ttl_seconds: 3600

  # Audit logging of mask/unmask operations (never logs original PII values at INFO)
  audit:
    enabled: true
    log_level: INFO