diff --git a/config.example.yaml b/config.example.yaml index 3da9f2e..c73e399 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -15,6 +15,19 @@ agent: # chat = direct text-only dataset generation via an OpenAI-compatible API provider: pi + # Trace each agent session to Langfuse (https://langfuse.com). Works for Codex + # and Claude Code -- each uses its own native integration (Codex plugin, Claude + # Code Stop hook) and Teich passes the credentials into the container. Side- + # channel only: it reads transcripts, fails open, and doesn't change agent + # behavior or Teich's output files. All three credentials are required when + # enabled. base_url may be Langfuse Cloud or a self-hosted URL; for a host-local + # instance use http://host.docker.internal:. + # langfuse: + # enabled: true + # public_key: pk-lf-... + # secret_key: sk-lf-... + # base_url: https://cloud.langfuse.com + model: # Model id passed to the selected agent/provider. model: deepseek/deepseek-v4-flash diff --git a/docker/codex-runtime.Dockerfile b/docker/codex-runtime.Dockerfile index 91c4ca9..31a6f65 100644 --- a/docker/codex-runtime.Dockerfile +++ b/docker/codex-runtime.Dockerfile @@ -47,6 +47,28 @@ RUN --mount=type=cache,target=/root/.cache/uv \ chmod +x /usr/local/bin/hermes && \ (hermes --version || hermes --help >/dev/null) +# Bake the Langfuse Codex observability plugin into a staging CODEX_HOME so Teich +# can seed it into each session offline (no per-run network on the sandboxes). +# Only used when agent.langfuse.enabled is set; side-channel/observability +# only. NOTE: pulls the plugin at its current HEAD -- the version is pinned at +# image-build time, not by a commit hash. chmod a+rX so the unprivileged in- +# container `codex` user (and the host `docker cp`) can read it. +RUN HOME=/opt/codex-langfuse CODEX_HOME=/opt/codex-langfuse/.codex \ + sh -c 'mkdir -p "$CODEX_HOME" \ + && codex plugin marketplace add langfuse/codex-observability-plugin \ + && codex plugin add tracing@codex-observability-plugin \ + && chmod -R a+rX /opt/codex-langfuse' + +# Langfuse hook + SDK for Claude Code, pinned to a known-good commit so the +# baked hook script is reproducible. The SDK goes in the venv because Claude +# strips it from a hook's PATH, so the hook calls it by full path. +RUN /opt/venv/bin/pip install --no-cache-dir "langfuse>=4.0,<5" && \ + git clone https://github.com/langfuse/Claude-Observability-Plugin.git \ + /opt/claude-langfuse-plugin && \ + git -C /opt/claude-langfuse-plugin checkout 597af67d6c6b369f3e55db6cfa2ebe444f1af46c && \ + rm -rf /opt/claude-langfuse-plugin/.git && \ + chmod -R a+rX /opt/claude-langfuse-plugin + # Create working directory and user in one layer WORKDIR /workspace RUN useradd -m -s /bin/bash codex && \ diff --git a/src/teich/cli.py b/src/teich/cli.py index 84f4e6f..c3a96e4 100644 --- a/src/teich/cli.py +++ b/src/teich/cli.py @@ -716,10 +716,20 @@ def generate( try: if agent_provider == "codex": runner = CodexRunner(cfg) + if cfg.agent.langfuse.enabled: + console.print( + "[cyan]Codex Langfuse tracing enabled: uploading each session to " + f"{cfg.agent.langfuse.base_url}.[/cyan]" + ) elif agent_provider == "pi": runner = PiRunner(cfg) elif agent_provider in {"claude", "claude-code", "claude_code"}: runner = ClaudeCodeRunner(cfg) + if cfg.agent.langfuse.enabled: + console.print( + "[cyan]Claude Code Langfuse tracing enabled: uploading each session to " + f"{cfg.agent.langfuse.base_url}.[/cyan]" + ) elif agent_provider in {"hermes", "hermes-agent", "hermes_agent"}: runner = HermesRunner(cfg) elif agent_provider == "chat": @@ -1021,6 +1031,19 @@ def init( # chat = direct text-only dataset generation via an OpenAI-compatible API provider: pi + # Trace each agent session to Langfuse (https://langfuse.com). Works for Codex + # and Claude Code -- each uses its own native integration (Codex plugin, Claude + # Code Stop hook) and Teich passes the credentials into the container. Side- + # channel only: it reads transcripts, fails open, and doesn't change agent + # behavior or Teich's output files. All three credentials are required when + # enabled. base_url may be Langfuse Cloud or a self-hosted URL; for a host-local + # instance use http://host.docker.internal:. + # langfuse: + # enabled: true + # public_key: pk-lf-... + # secret_key: sk-lf-... + # base_url: https://cloud.langfuse.com + model: # Model id passed to the selected agent/provider. model: deepseek/deepseek-v4-flash diff --git a/src/teich/config.py b/src/teich/config.py index 358fc3e..82bacae 100644 --- a/src/teich/config.py +++ b/src/teich/config.py @@ -96,9 +96,43 @@ class APIConfig(BaseModel): wire_api: str = "responses" +class LangfuseConfig(BaseModel): + """Langfuse tracing credentials, set under ``agent.langfuse``. + + When enabled, Teich wires each agent's Langfuse integration (the Codex + plugin, the Claude Code Stop hook) and passes these credentials into the + container. Tracing is side-channel only: it reads transcripts, fails open, + and does not change agent behavior or Teich's output files. + """ + enabled: bool = False + public_key: str | None = None + secret_key: str | None = None + base_url: str | None = None + + @model_validator(mode="after") + def require_credentials_when_enabled(self) -> LangfuseConfig: + if self.enabled: + missing = [ + name + for name, value in ( + ("public_key", self.public_key), + ("secret_key", self.secret_key), + ("base_url", self.base_url), + ) + if not (value and value.strip()) + ] + if missing: + raise ValueError( + "langfuse requires " + ", ".join(missing) + " when enabled" + ) + return self + + class AgentConfig(BaseModel): """Agent runtime selection.""" provider: str = "codex" + # Langfuse tracing, applied to every agent that supports it (Codex, Claude). + langfuse: LangfuseConfig = Field(default_factory=LangfuseConfig) class ModelConfig(BaseModel): diff --git a/src/teich/runner.py b/src/teich/runner.py index c3fec17..ddb4a0d 100644 --- a/src/teich/runner.py +++ b/src/teich/runner.py @@ -45,6 +45,21 @@ CODEX_HOME_IN_CONTAINER = "/home/codex/.codex" CLAUDE_HOME_IN_CONTAINER = "/home/codex/.claude" HERMES_HOME_IN_CONTAINER = "/home/codex/.hermes" + +# Langfuse Codex observability plugin: baked into the image at a staging +# CODEX_HOME (see codex-runtime.Dockerfile) so Teich can install it into each +# session's CODEX_HOME offline -- no per-run network. Only the ``plugins`` tree +# and the config.toml ``[plugins."tracing@..."]`` enable entry are needed at run +# time; the marketplace snapshot is only used for install/upgrade. +LANGFUSE_PLUGIN_STAGE_CODEX_HOME = "/opt/codex-langfuse/.codex" +LANGFUSE_PLUGIN_ID = "tracing@codex-observability-plugin" + +# Claude strips /opt/venv from a hook's PATH, so call the venv python (which has +# the langfuse SDK) by absolute path. Script is baked in by the Dockerfile. +CLAUDE_LANGFUSE_PLUGIN_DIR = "/opt/claude-langfuse-plugin" +CLAUDE_LANGFUSE_HOOK_COMMAND = ( + f"/opt/venv/bin/python3 {CLAUDE_LANGFUSE_PLUGIN_DIR}/hooks/langfuse_hook.py" +) PI_AGENT_DIR_IN_CONTAINER = "/home/codex/.pi/agent" PI_SESSIONS_DIR_IN_CONTAINER = "/home/codex/pi-sessions" WORKSPACE_IN_CONTAINER = "/workspace" @@ -1022,6 +1037,19 @@ def _container_base_url(base_url: str | None) -> str | None: netloc = parsed.netloc.replace(hostname, "host.docker.internal", 1) return urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment)) + def _langfuse_container_base_url(self) -> str | None: + """The Langfuse base_url as seen from inside the container (localhost -> + host.docker.internal).""" + return self._container_base_url(self.config.agent.langfuse.base_url) + + def _langfuse_is_host_local(self) -> bool: + """Whether tracing points at the host, so the container needs + --add-host host.docker.internal.""" + langfuse = self.config.agent.langfuse + return langfuse.enabled and "host.docker.internal" in ( + self._langfuse_container_base_url() or "" + ) + @staticmethod def _prompt_preview(prompt: str, limit: int = 60) -> str: normalized = " ".join(prompt.split()) @@ -2006,6 +2034,11 @@ def _run_process( class CodexRunner(DockerRuntimeRunner): """Manages Docker-based Codex sessions.""" + def __init__(self, config: Config): + self._langfuse_plugin_cache: Path | None = None + self._langfuse_plugin_lock = threading.Lock() + super().__init__(config) + @staticmethod def _toml_string(value: str) -> str: return json.dumps(value) @@ -2082,6 +2115,72 @@ def _is_likely_incompatible_custom_provider(provider: str) -> bool: normalized = provider.strip().lower() return normalized in {"llama.cpp", "llama_cpp", "llamacpp"} + def _ensure_langfuse_plugin_cache(self) -> Path | None: + """Extract the image-baked Langfuse plugin once per run. + + Returns the host cache dir (containing a ``plugins`` subtree), or ``None`` + when Langfuse tracing is disabled. The plugin is baked into the runtime + image; we copy it out once via ``docker cp`` so each session can be seeded + offline. ``run_session`` runs from a thread pool, so guard creation with a + double-checked lock. + """ + if not self.config.agent.langfuse.enabled: + return None + if self._langfuse_plugin_cache is None: + with self._langfuse_plugin_lock: + if self._langfuse_plugin_cache is None: + self._ensure_image() + cache_dir = Path(tempfile.mkdtemp(prefix="teich-langfuse-plugin-")) + self._extract_langfuse_plugin(cache_dir) + self._langfuse_plugin_cache = cache_dir + return self._langfuse_plugin_cache + + def _extract_langfuse_plugin(self, cache_dir: Path) -> None: + """Copy the baked ``plugins`` tree out of the runtime image into ``cache_dir``.""" + created = subprocess.run( + ["docker", "create", self.image_name], + capture_output=True, + text=True, + check=True, + ) + container_id = created.stdout.strip() + try: + subprocess.run( + [ + "docker", + "cp", + f"{container_id}:{LANGFUSE_PLUGIN_STAGE_CODEX_HOME}/plugins", + str(cache_dir / "plugins"), + ], + capture_output=True, + text=True, + check=True, + ) + finally: + subprocess.run( + ["docker", "rm", "-f", container_id], + capture_output=True, + text=True, + check=False, + ) + + def _install_codex_langfuse_plugin(self, codex_home: Path) -> None: + """Seed a session's CODEX_HOME with the Langfuse plugin tree. + + No-op when Langfuse is disabled. Files are made world-readable/traversable + so the in-container ``codex`` user can execute the hook regardless of how + its uid maps to the host. + """ + cache = self._ensure_langfuse_plugin_cache() + if cache is None: + return + source = cache / "plugins" + dest = codex_home / "plugins" + shutil.copytree(source, dest, dirs_exist_ok=True) + for path in dest.rglob("*"): + path.chmod(0o777 if path.is_dir() else 0o644) + dest.chmod(0o777) + def _write_codex_config(self, codex_home: Path) -> None: codex_home.mkdir(parents=True, exist_ok=True) codex_home.chmod(0o777) @@ -2138,6 +2237,22 @@ def _write_codex_config(self, codex_home: Path) -> None: for key, value in mcp.env.items(): lines.append(f"{self._toml_string(key)} = {self._toml_string(value)}") + if self.config.agent.langfuse.enabled: + # Enable the plugin-hooks feature and the (image-baked, offline) + # Langfuse tracing plugin. The plugin tree itself is copied into + # CODEX_HOME by _install_codex_langfuse_plugin. Tables go after the + # top-level keys above, keeping the TOML valid. + lines.extend( + [ + "", + "[features]", + "plugin_hooks = true", + "", + f'[plugins."{LANGFUSE_PLUGIN_ID}"]', + "enabled = true", + ] + ) + config_text = "\n".join(lines).strip() config_path = codex_home / "config.toml" config_path.write_text( @@ -2283,11 +2398,33 @@ def _build_codex_docker_base_command( "-w", WORKSPACE_IN_CONTAINER, ] - if proxy_target or (configured_base_url and base_url != configured_base_url): + langfuse = self.config.agent.langfuse + langfuse_base_url = self._langfuse_container_base_url() + if ( + proxy_target + or self._langfuse_is_host_local() + or (configured_base_url and base_url != configured_base_url) + ): cmd.extend([ "--add-host", "host.docker.internal:host-gateway", ]) + if langfuse.enabled: + # The codex-observability-plugin Stop hook uploads each session + # transcript to Langfuse. Side-channel only; the plugin tree is seeded + # into CODEX_HOME by _install_codex_langfuse_plugin. + cmd.extend( + [ + "-e", + "TRACE_TO_LANGFUSE=true", + "-e", + f"LANGFUSE_PUBLIC_KEY={langfuse.public_key}", + "-e", + f"LANGFUSE_SECRET_KEY={langfuse.secret_key}", + "-e", + f"LANGFUSE_BASE_URL={langfuse_base_url or ''}", + ] + ) if proxy_target: cmd.extend( [ @@ -2326,6 +2463,11 @@ def _build_codex_agent_command(self, resume: bool = False) -> list[str]: codex_cmd.extend(["resume", "--last"]) codex_cmd.extend(["--model", model]) codex_cmd.append("--skip-git-repo-check") + if self.config.agent.langfuse.enabled: + # Codex trust-gates plugin hooks in non-interactive `exec` and + # silently skips them otherwise. Teich vets and bakes the Langfuse + # plugin itself, so bypass the persisted-trust requirement. + codex_cmd.append("--dangerously-bypass-hook-trust") if base_url and not self._is_oss_local_provider(provider_name): provider_key = self._custom_provider_key(provider_name) provider_literal = ( @@ -2476,6 +2618,7 @@ def run_session( try: self._write_codex_config(codex_home) + self._install_codex_langfuse_plugin(codex_home) if self._local_provider_proxy_target(self.config.api.provider, self.config.get_base_url()): self._write_local_provider_proxy(codex_home) existing_sessions = {path.resolve() for path in self._list_session_files(codex_home)} @@ -2609,12 +2752,17 @@ def run_all( prompt_inputs: list[PromptInput] | None = None, resume: bool = False, ) -> list[Path]: - return super().run_all( - max_concurrency=max_concurrency, - progress_callback=progress_callback, - prompt_inputs=prompt_inputs, - resume=resume, - ) + try: + return super().run_all( + max_concurrency=max_concurrency, + progress_callback=progress_callback, + prompt_inputs=prompt_inputs, + resume=resume, + ) + finally: + if self._langfuse_plugin_cache is not None: + shutil.rmtree(self._langfuse_plugin_cache, ignore_errors=True) + self._langfuse_plugin_cache = None class ExternalCliRunner(DockerRuntimeRunner): @@ -2667,6 +2815,15 @@ def _base_url_env_items(self) -> list[tuple[str, str]]: ("ANTHROPIC_BASE_URL", base_url), ] + def _langfuse_env_items(self) -> list[tuple[str, str]]: + """Langfuse env vars to pass into the container. Empty by default; + agents that support tracing override this.""" + return [] + + def _prepare_agent_home(self, home_dir: Path) -> None: + """Seed the per-session agent home before the container runs. No-op by + default; agents override to write settings/config (e.g. Langfuse hooks).""" + def _build_external_docker_base_command( self, workspace: Path, @@ -2696,9 +2853,17 @@ def _build_external_docker_base_command( WORKSPACE_IN_CONTAINER, ] configured_base_url = self.config.get_base_url() - if configured_base_url and self._container_base_url(configured_base_url) != configured_base_url: + base_url_is_host_local = bool( + configured_base_url + and self._container_base_url(configured_base_url) != configured_base_url + ) + if base_url_is_host_local or self._langfuse_is_host_local(): command.extend(["--add-host", "host.docker.internal:host-gateway"]) - for key, value in [*self._api_env_items(), *self._base_url_env_items()]: + for key, value in [ + *self._api_env_items(), + *self._base_url_env_items(), + *self._langfuse_env_items(), + ]: command.extend(["-e", f"{key}={value}"]) command.append(self.image_name) return command @@ -2901,6 +3066,7 @@ def run_session( workspace_root, workspace = self._prepare_workspace(session_id, prompt_input, self.container_kind) home_dir = Path(tempfile.mkdtemp(prefix=f"{self.container_kind}-home-{session_id}-")) home_dir.chmod(0o777) + self._prepare_agent_home(home_dir) started_at = datetime.now(timezone.utc) container_name = self._container_name(self.container_kind, session_id) turn_prompts = _agent_turn_prompts(prompt, prompt_input) @@ -2973,6 +3139,27 @@ class ClaudeCodeRunner(ExternalCliRunner): source_name = "claude-code" default_model_provider = "anthropic" + def _langfuse_env_items(self) -> list[tuple[str, str]]: + langfuse = self.config.agent.langfuse + if not langfuse.enabled: + return [] + return [ + ("TRACE_TO_LANGFUSE", "true"), + ("LANGFUSE_PUBLIC_KEY", langfuse.public_key or ""), + ("LANGFUSE_SECRET_KEY", langfuse.secret_key or ""), + ("LANGFUSE_BASE_URL", self._langfuse_container_base_url() or ""), + ] + + def _prepare_agent_home(self, home_dir: Path) -> None: + """Register the Langfuse Stop/SessionEnd hook when tracing is enabled.""" + if not self.config.agent.langfuse.enabled: + return + hook = {"hooks": [{"type": "command", "command": CLAUDE_LANGFUSE_HOOK_COMMAND}]} + settings = {"hooks": {"Stop": [hook], "SessionEnd": [hook]}} + settings_path = home_dir / "settings.json" + settings_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") + settings_path.chmod(0o666) + @staticmethod def _list_native_session_files(home_dir: Path) -> list[Path]: projects_dir = home_dir / "projects" @@ -3305,6 +3492,7 @@ def run_session( workspace_root, workspace = self._prepare_workspace(session_id, prompt_input, self.container_kind) home_dir = Path(tempfile.mkdtemp(prefix=f"{self.container_kind}-home-{session_id}-")) home_dir.chmod(0o777) + self._prepare_agent_home(home_dir) started_at = datetime.now(timezone.utc) container_name = self._container_name(self.container_kind, session_id) turn_prompts = _agent_turn_prompts(prompt, prompt_input) @@ -3839,6 +4027,7 @@ def run_session( workspace_root, workspace = self._prepare_workspace(session_id, prompt_input, self.container_kind) home_dir = Path(tempfile.mkdtemp(prefix=f"{self.container_kind}-home-{session_id}-")) home_dir.chmod(0o777) + self._prepare_agent_home(home_dir) container_name = self._container_name(self.container_kind, session_id) turn_prompts = _agent_turn_prompts(prompt, prompt_input) fallback_destination = self._resolve_hermes_trace_path() diff --git a/tests/test_codex_langfuse.py b/tests/test_codex_langfuse.py new file mode 100644 index 0000000..c92e59a --- /dev/null +++ b/tests/test_codex_langfuse.py @@ -0,0 +1,165 @@ +"""Tests for Codex -> Langfuse tracing wiring. + +These tests are hermetic: they never touch Docker, the network, or a real +Langfuse instance. They cover the config.toml blocks Teich writes to enable the +plugin, the Langfuse env vars passed to the container, the exec hook-trust flag, +and the per-session install of the (image-baked) plugin tree into CODEX_HOME. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from teich.config import Config, ModelConfig +from teich.runner import CodexRunner + + +def _langfuse_config(**overrides) -> Config: + langfuse = { + "enabled": True, + "public_key": "pk-lf-123", + "secret_key": "sk-lf-456", + "base_url": "https://langfuse.example.com", + } + langfuse.update(overrides) + return Config( + model=ModelConfig(model="gpt-5.5"), + agent={"provider": "codex", "langfuse": langfuse}, + ) + + +# -- config.toml blocks ------------------------------------------------------ + +def test_codex_config_writes_langfuse_blocks_when_enabled(tmp_path: Path): + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(_langfuse_config()) + codex_home = tmp_path / ".codex" + runner._write_codex_config(codex_home) + content = (codex_home / "config.toml").read_text(encoding="utf-8") + assert "[features]" in content + assert "plugin_hooks = true" in content + assert '[plugins."tracing@codex-observability-plugin"]' in content + assert "enabled = true" in content + + +def test_codex_config_omits_langfuse_blocks_when_disabled(tmp_path: Path): + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(Config(model=ModelConfig(model="gpt-5.5"))) + codex_home = tmp_path / ".codex" + runner._write_codex_config(codex_home) + content = (codex_home / "config.toml").read_text(encoding="utf-8") + assert "plugin_hooks" not in content + assert "tracing@codex-observability-plugin" not in content + + +# -- container env vars ------------------------------------------------------ + +def test_codex_command_passes_langfuse_env_when_enabled(tmp_path: Path): + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(_langfuse_config()) + cmd = runner._build_codex_command( + "Build app", + workspace=tmp_path / "ws", + codex_home=tmp_path / "ch", + container_name="teich-codex-x", + ) + assert "TRACE_TO_LANGFUSE=true" in cmd + assert "LANGFUSE_PUBLIC_KEY=pk-lf-123" in cmd + assert "LANGFUSE_SECRET_KEY=sk-lf-456" in cmd + assert "LANGFUSE_BASE_URL=https://langfuse.example.com" in cmd + + +def test_codex_command_omits_langfuse_env_when_disabled(tmp_path: Path): + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(Config(model=ModelConfig(model="gpt-5.5"))) + cmd = runner._build_codex_command( + "Build app", + workspace=tmp_path / "ws", + codex_home=tmp_path / "ch", + container_name="teich-codex-x", + ) + assert not any(part.startswith("TRACE_TO_LANGFUSE=") for part in cmd) + assert not any(part.startswith("LANGFUSE_") for part in cmd) + + +@pytest.mark.parametrize("host", ["localhost", "127.0.0.1"]) +def test_codex_command_rewrites_host_local_langfuse_base_url(tmp_path: Path, host: str): + cfg = _langfuse_config(base_url=f"http://{host}:3000") + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(cfg) + cmd = runner._build_codex_command( + "Build app", + workspace=tmp_path / "ws", + codex_home=tmp_path / "ch", + container_name="teich-codex-x", + ) + assert "LANGFUSE_BASE_URL=http://host.docker.internal:3000" in cmd + assert "host.docker.internal:host-gateway" in cmd + + +def test_codex_command_adds_host_gateway_for_host_local_langfuse(tmp_path: Path): + cfg = _langfuse_config(base_url="http://host.docker.internal:3000") + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(cfg) + cmd = runner._build_codex_command( + "Build app", + workspace=tmp_path / "ws", + codex_home=tmp_path / "ch", + container_name="teich-codex-x", + ) + assert "host.docker.internal:host-gateway" in cmd + + +# -- hook-trust bypass (exec) ------------------------------------------------ + +def test_codex_agent_command_bypasses_hook_trust_when_enabled(): + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(_langfuse_config()) + cmd = runner._build_codex_agent_command() + assert "exec" in cmd + assert "--dangerously-bypass-hook-trust" in cmd + + +def test_codex_agent_command_no_hook_trust_bypass_when_disabled(): + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(Config(model=ModelConfig(model="gpt-5.5"))) + cmd = runner._build_codex_agent_command() + assert "--dangerously-bypass-hook-trust" not in cmd + + +# -- per-session plugin install --------------------------------------------- + +def test_install_codex_langfuse_plugin_copies_tree(tmp_path: Path): + # Fake the image-baked cache so the test never touches Docker. + cache = tmp_path / "cache" + leaf = cache / "plugins" / "cache" / "codex-observability-plugin" / "tracing" / "0.1.0" / "dist" + leaf.mkdir(parents=True) + (leaf / "index.mjs").write_text("// bundle", encoding="utf-8") + + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(_langfuse_config()) + codex_home = tmp_path / ".codex" + codex_home.mkdir() + + with patch.object(runner, "_ensure_langfuse_plugin_cache", return_value=cache): + runner._install_codex_langfuse_plugin(codex_home) + + installed = ( + codex_home / "plugins" / "cache" / "codex-observability-plugin" + / "tracing" / "0.1.0" / "dist" / "index.mjs" + ) + assert installed.exists() + assert installed.read_text(encoding="utf-8") == "// bundle" + + +def test_install_codex_langfuse_plugin_noop_when_disabled(tmp_path: Path): + with patch.object(CodexRunner, "_ensure_image"): + runner = CodexRunner(Config(model=ModelConfig(model="gpt-5.5"))) + codex_home = tmp_path / ".codex" + codex_home.mkdir() + # Disabled -> cache is None -> nothing copied, no error. + runner._install_codex_langfuse_plugin(codex_home) + assert not (codex_home / "plugins").exists() diff --git a/tests/test_langfuse_agents.py b/tests/test_langfuse_agents.py new file mode 100644 index 0000000..32eefbe --- /dev/null +++ b/tests/test_langfuse_agents.py @@ -0,0 +1,118 @@ +"""Tests for the shared Langfuse config + Claude Code tracing wiring. + +Hermetic: no Docker, network, or real Langfuse. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +from teich.config import Config, LangfuseConfig, ModelConfig +from teich.runner import ClaudeCodeRunner + + +# -- shared config ----------------------------------------------------------- + +def test_langfuse_disabled_by_default(): + assert Config().agent.langfuse.enabled is False + + +def test_langfuse_ok_with_all_credentials(): + cfg = LangfuseConfig(enabled=True, public_key="pk", secret_key="sk", base_url="https://x") + assert cfg.enabled and cfg.public_key == "pk" + + +@pytest.mark.parametrize("missing", ["public_key", "secret_key", "base_url"]) +def test_langfuse_requires_each_credential(missing: str): + kwargs = {"public_key": "pk", "secret_key": "sk", "base_url": "https://x"} + del kwargs[missing] + with pytest.raises(ValueError, match=missing): + LangfuseConfig(enabled=True, **kwargs) + + +@pytest.mark.parametrize("blank", ["public_key", "secret_key", "base_url"]) +def test_langfuse_rejects_blank_credential(blank: str): + kwargs = {"public_key": "pk", "secret_key": "sk", "base_url": "https://x", blank: " "} + with pytest.raises(ValueError, match=blank): + LangfuseConfig(enabled=True, **kwargs) + + +# -- Claude Code env items --------------------------------------------------- + +def _claude_langfuse_config(base_url: str = "https://langfuse.example.com") -> Config: + return Config( + model=ModelConfig(model="claude-sonnet-4-6"), + agent={ + "provider": "claude-code", + "langfuse": { + "enabled": True, + "public_key": "pk-lf-1", + "secret_key": "sk-lf-2", + "base_url": base_url, + }, + }, + ) + + +def test_claude_langfuse_env_items_when_enabled(): + with patch.object(ClaudeCodeRunner, "_ensure_image"): + runner = ClaudeCodeRunner(_claude_langfuse_config()) + items = dict(runner._langfuse_env_items()) + assert items["TRACE_TO_LANGFUSE"] == "true" + assert items["LANGFUSE_PUBLIC_KEY"] == "pk-lf-1" + assert items["LANGFUSE_SECRET_KEY"] == "sk-lf-2" + assert items["LANGFUSE_BASE_URL"] == "https://langfuse.example.com" + + +def test_claude_langfuse_env_items_empty_when_disabled(): + with patch.object(ClaudeCodeRunner, "_ensure_image"): + runner = ClaudeCodeRunner(Config(model=ModelConfig(model="claude-sonnet-4-6"))) + assert runner._langfuse_env_items() == [] + + +# -- Claude settings.json hook ----------------------------------------------- + +def test_claude_prepare_home_writes_stop_hook(tmp_path: Path): + with patch.object(ClaudeCodeRunner, "_ensure_image"): + runner = ClaudeCodeRunner(_claude_langfuse_config()) + home = tmp_path / "home" + home.mkdir() + runner._prepare_agent_home(home) + settings = json.loads((home / "settings.json").read_text()) + for event in ("Stop", "SessionEnd"): + cmd = settings["hooks"][event][0]["hooks"][0]["command"] + # Must use the venv python by absolute path (claude sanitizes PATH for hooks). + assert cmd.startswith("/opt/venv/bin/python3 ") + assert cmd.endswith("langfuse_hook.py") + + +def test_claude_prepare_home_noop_when_disabled(tmp_path: Path): + with patch.object(ClaudeCodeRunner, "_ensure_image"): + runner = ClaudeCodeRunner(Config(model=ModelConfig(model="claude-sonnet-4-6"))) + home = tmp_path / "home" + home.mkdir() + runner._prepare_agent_home(home) + assert not (home / "settings.json").exists() + + +# -- host-local base_url rewriting ------------------------------------------- + +@pytest.mark.parametrize("host", ["localhost", "127.0.0.1"]) +def test_claude_langfuse_base_url_rewrites_host_local(host: str): + with patch.object(ClaudeCodeRunner, "_ensure_image"): + runner = ClaudeCodeRunner(_claude_langfuse_config(base_url=f"http://{host}:3000")) + items = dict(runner._langfuse_env_items()) + assert items["LANGFUSE_BASE_URL"] == "http://host.docker.internal:3000" + assert runner._langfuse_is_host_local() is True + + +def test_cloud_base_url_is_not_rewritten(): + with patch.object(ClaudeCodeRunner, "_ensure_image"): + runner = ClaudeCodeRunner(_claude_langfuse_config()) + items = dict(runner._langfuse_env_items()) + assert items["LANGFUSE_BASE_URL"] == "https://langfuse.example.com" + assert runner._langfuse_is_host_local() is False