From 3847dfc5312d8e2389c91ed367e07fbeda9f4f02 Mon Sep 17 00:00:00 2001 From: Niv Yehezkel Date: Mon, 25 May 2026 01:05:40 +0300 Subject: [PATCH] feat: add support for LLM gateways --- docs/pages/deploying-in-production.mdx | 22 +++++ services/api/api/tool_manager.py | 103 +++++++++++++----------- services/api/tests/test_tool_manager.py | 26 ++++++ 3 files changed, 103 insertions(+), 48 deletions(-) diff --git a/docs/pages/deploying-in-production.mdx b/docs/pages/deploying-in-production.mdx index f30d44b11..d8e18a527 100644 --- a/docs/pages/deploying-in-production.mdx +++ b/docs/pages/deploying-in-production.mdx @@ -98,6 +98,28 @@ so any thread can use any configured credential. Per-user and per-channel scoping is on the roadmap; until then, scope tool and harness access accordingly. See [Security](/security) for the full threat model. +### Optional: route LLM traffic through a gateway + +To send harness LLM calls through a gateway (LiteLLM, Portkey, self-hosted, etc.) +instead of directly to provider APIs, set `CENTAUR_LLM_GATEWAY_HOST` on the API +container and point the harness CLI at the gateway via `*_BASE_URL` on the +sandbox. Iron-proxy injects `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` for the +gateway host instead of the provider host — store the gateway's API key under +the existing `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` name. + +```yaml +api: + extraEnv: + CENTAUR_LLM_GATEWAY_HOST: "litellm.internal.example.com" # hostname only + +sandbox: + extraEnv: + ANTHROPIC_BASE_URL: "https://litellm.internal.example.com" # full URL + # or OPENAI_BASE_URL for the Codex harness +``` + +When unset, behavior is unchanged: keys route to the upstream provider hosts. + ## 4. Configure Slack Create the Slackbot app at [api.slack.com/apps](https://api.slack.com/apps). diff --git a/services/api/api/tool_manager.py b/services/api/api/tool_manager.py index 58e73f089..c60ce7951 100644 --- a/services/api/api/tool_manager.py +++ b/services/api/api/tool_manager.py @@ -20,7 +20,7 @@ from dataclasses import asdict, dataclass, is_dataclass from enum import Enum from pathlib import Path -from typing import Any, ClassVar +from typing import Any import structlog from fastapi import APIRouter, Depends, HTTPException, Request @@ -1589,52 +1589,59 @@ def discover(self) -> list[LoadedTool]: ) return loaded - # Hardcoded infrastructure secrets for the injection map. Each ``HttpSecret`` - # carries the hosts iron-proxy attaches it to. - _INFRA_SECRETS: ClassVar[list[HttpSecret]] = [ - HttpSecret( - name="ANTHROPIC_API_KEY", - secret_ref="ANTHROPIC_API_KEY", - hosts=("api.anthropic.com",), - match_headers=("X-Api-Key",), - ), - HttpSecret( - name="OPENAI_API_KEY", - secret_ref="OPENAI_API_KEY", - hosts=("api.openai.com",), - match_headers=("Authorization",), - ), - HttpSecret( - name="XAI_API_KEY", - secret_ref="XAI_API_KEY", - hosts=("api.x.ai",), - match_headers=("Authorization",), - ), - HttpSecret( - name="GEMINI_API_KEY", - secret_ref="GEMINI_API_KEY", - hosts=("generativelanguage.googleapis.com",), - match_headers=("X-Goog-Api-Key",), - ), - HttpSecret( - name="AMP_API_KEY", - secret_ref="AMP_API_KEY", - hosts=("ampcode.com",), - match_headers=("Authorization",), - ), - HttpSecret( - name="GITHUB_TOKEN", - secret_ref="GITHUB_TOKEN", - hosts=("github.com", "api.github.com"), - match_headers=("Authorization",), - ), - HttpSecret( - name="SLACK_BOT_TOKEN", - secret_ref="SLACK_BOT_TOKEN", - hosts=("*.slack.com",), - match_headers=("Authorization",), - ), - ] + # Infrastructure secrets for the injection map. Each ``HttpSecret`` carries + # the hosts iron-proxy attaches it to. When ``CENTAUR_LLM_GATEWAY_HOST`` is + # set, the LLM provider keys are also injected on outbound calls to that + # host, so harnesses can be pointed at a LiteLLM-style gateway via + # ``ANTHROPIC_BASE_URL`` / ``OPENAI_BASE_URL`` without leaking placeholders. + def _infra_secrets(self) -> list[HttpSecret]: + gateway = os.getenv("CENTAUR_LLM_GATEWAY_HOST", "").strip() + anthropic_hosts: tuple[str, ...] = (gateway,) if gateway else ("api.anthropic.com",) + openai_hosts: tuple[str, ...] = (gateway,) if gateway else ("api.openai.com",) + return [ + HttpSecret( + name="ANTHROPIC_API_KEY", + secret_ref="ANTHROPIC_API_KEY", + hosts=anthropic_hosts, + match_headers=("X-Api-Key",), + ), + HttpSecret( + name="OPENAI_API_KEY", + secret_ref="OPENAI_API_KEY", + hosts=openai_hosts, + match_headers=("Authorization",), + ), + HttpSecret( + name="XAI_API_KEY", + secret_ref="XAI_API_KEY", + hosts=("api.x.ai",), + match_headers=("Authorization",), + ), + HttpSecret( + name="GEMINI_API_KEY", + secret_ref="GEMINI_API_KEY", + hosts=("generativelanguage.googleapis.com",), + match_headers=("X-Goog-Api-Key",), + ), + HttpSecret( + name="AMP_API_KEY", + secret_ref="AMP_API_KEY", + hosts=("ampcode.com",), + match_headers=("Authorization",), + ), + HttpSecret( + name="GITHUB_TOKEN", + secret_ref="GITHUB_TOKEN", + hosts=("github.com", "api.github.com"), + match_headers=("Authorization",), + ), + HttpSecret( + name="SLACK_BOT_TOKEN", + secret_ref="SLACK_BOT_TOKEN", + hosts=("*.slack.com",), + match_headers=("Authorization",), + ), + ] def collect_secrets(self) -> list[SecretDef]: """Return all secrets (infra + tool). @@ -1642,7 +1649,7 @@ def collect_secrets(self) -> list[SecretDef]: Every ``HttpSecret``, ``GcpAuthSecret`` and ``OAuthTokenSecret`` carries its own ``hosts``; ``PgDsnSecret`` is a TCP listener with no host. """ - out: list[SecretDef] = list(self._INFRA_SECRETS) + out: list[SecretDef] = list(self._infra_secrets()) for lt in self.tools.values(): out.extend(lt.all_secrets) return out diff --git a/services/api/tests/test_tool_manager.py b/services/api/tests/test_tool_manager.py index 5f153b6d0..ac39d74f6 100644 --- a/services/api/tests/test_tool_manager.py +++ b/services/api/tests/test_tool_manager.py @@ -660,3 +660,29 @@ async def test_tool_rest_router_lists_describes_and_invokes_tools( '{"error": "Method \'missing\' not found in tool \'alpha\'", ' '"available_methods": ["async_echo", "secret_values", "sync_echo"]}' ) + + +def _llm_hosts(manager: ToolManager) -> dict[str, tuple[str, ...]]: + return {s.name: s.hosts for s in manager._infra_secrets() if s.name in {"ANTHROPIC_API_KEY", "OPENAI_API_KEY"}} + + +def test_infra_secrets_default_to_provider_hosts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("CENTAUR_LLM_GATEWAY_HOST", raising=False) + hosts = _llm_hosts(ToolManager(tmp_path)) + assert hosts == { + "ANTHROPIC_API_KEY": ("api.anthropic.com",), + "OPENAI_API_KEY": ("api.openai.com",), + } + + +def test_infra_secrets_route_llm_keys_through_gateway_host( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CENTAUR_LLM_GATEWAY_HOST", "litellm.example.internal") + hosts = _llm_hosts(ToolManager(tmp_path)) + assert hosts == { + "ANTHROPIC_API_KEY": ("litellm.example.internal",), + "OPENAI_API_KEY": ("litellm.example.internal",), + }