[codex] Fix runner trace matrix and Hermes detection

examples/prompts.jsonl

@@ -1,3 +1,4 @@
+{"prompt": "I'm a data scientist at my job where we sequence large amounts of genetic data to calculate the likelihood of genetic disease and even predict genetic mutations, tasked with using machine learning to solve a problem. I have a dataset and a general idea of what I want to do, but I'm not sure how to get started. Put together a step-by-step plan for how I can approach this problem, including data preprocessing, model selection, training, and evaluation. Make sure to include any important considerations or potential pitfalls I should be aware of along the way.", "follow_up_prompts": ["Can you draw some mermaid diagrams to illustrate how this whole process should work?"]}
 {"prompt":"Make me windows 12, fully working apps and everything.","follow_up_prompts":["lots of mistakes, fix them all"]}
 {"prompt":"Make me a cool website for comparing a bunch of different LoRA finetunes that I make. It'd be nice to organize them all in one place.","follow_up_prompts":["You did alright but i want you to make it look better. Redo the whole UI and make it look professional like it's something made with Anthropic style. Sleek, modern, minimalistic, and featureful."]}
 {"prompt":"Make me a 3D platformer game in vanilla HTML/CSS/JS in one file. Keep it playable and don't use any external libraries.","follow_up_prompts":["Now catch any edge cases, fix anything broken, and make sure none of the code is busted."]}

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "teich"
-version = "0.1.3"
+version = "0.1.4"
 description = "Turn coding agent traces into auditable supervised fine-tuning data"
 readme = "README.md"
 license = {text = "Apache-2.0"}

src/teich/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__version__ = "0.1.3"
+__version__ = "0.1.4"
 
 from .audit import SFTAuditReport, audit_sft_dataset
 from .config import Config, load_config

src/teich/converter.py

@@ -1050,6 +1050,12 @@ def _is_openclaw_session_header(event: Any) -> bool:
     return isinstance(cwd, str) and ".openclaw" in cwd
 
 
+def _external_session_source(event: dict[str, Any]) -> str:
+    payload = event.get("payload")
+    source = payload.get("source") if isinstance(payload, dict) else None
+    return source.strip().lower() if isinstance(source, str) else ""
+
+
 def _detect_trace_type(events: list[Any], default: TraceType | None = "codex") -> TraceType | None:
     first_event = next((event for event in events if isinstance(event, dict)), None)
     if _is_openclaw_session_header(first_event):
@@ -1064,10 +1070,11 @@ def _detect_trace_type(events: list[Any], default: TraceType | None = "codex") -
             return "hermes"
         event_type = event.get("type")
         if event_type == "external_session_meta":
-            payload = event.get("payload")
-            source = payload.get("source") if isinstance(payload, dict) else None
-            if isinstance(source, str) and source.strip().lower() in {"claude", "claude-code", "claude_code"}:
+            source = _external_session_source(event)
+            if source in {"claude", "claude-code", "claude_code"}:
                 return "claude_code"
+            if source in {"hermes", "hermes-agent", "hermes_agent"}:
+                return "hermes"
             return "external_agent"
         if event_type == "hermes_session_meta":
             return "hermes"
@@ -1564,11 +1571,12 @@ def _convert_hermes_trace_to_training_example(
     for event in message_events:
         if not isinstance(event, dict):
             continue
-        if event.get("type") == "hermes_session_meta":
+        if event.get("type") in {"hermes_session_meta", "external_session_meta"}:
             payload = event.get("payload")
             if isinstance(payload, dict):
                 session_meta = payload
                 _add_explicit_tools(payload.get("tools"), explicit_tools, tool_names, tool_schemas)
+                _add_explicit_tools(payload.get("available_tools"), explicit_tools, tool_names, tool_schemas)
             continue
 
         role = event.get("role")
@@ -1650,13 +1658,15 @@ def _convert_hermes_trace_to_training_example(
         "source_file": trace_file.name,
         "session_id": session_meta.get("id") or trace_file.stem,
         "trace_type": "hermes",
+        "source": session_meta.get("source"),
         "teich_export_status": session_meta.get("teich_export_status"),
         "teich_partial": session_meta.get("teich_partial"),
         "model_provider": session_meta.get("model_provider"),
         "configured_model_provider": session_meta.get("configured_model_provider"),
         "model": session_meta.get("model"),
         "configured_context_length": session_meta.get("configured_context_length"),
         "cwd": session_meta.get("cwd"),
+        "cli_version": session_meta.get("cli_version"),
         "parent_session_id": session_meta.get("parent_session_id"),
         "hermes_source": session_meta.get("hermes_source"),
         "message_count": session_meta.get("message_count"),
@@ -1676,6 +1686,10 @@ def _convert_hermes_trace_to_training_example(
         "billing_base_url": session_meta.get("billing_base_url"),
         "billing_mode": session_meta.get("billing_mode"),
         "system_prompt": session_meta.get("system_prompt"),
+        "started_at": session_meta.get("started_at"),
+        "ended_at": session_meta.get("ended_at"),
+        "end_reason": session_meta.get("end_reason"),
+        "api_call_count": session_meta.get("api_call_count"),
         "turn_count": sum(1 for message in messages if message.get("role") == "user"),
     }
     _add_first_message_timestamp(metadata, first_message_timestamp)
@@ -2129,8 +2143,20 @@ def _convert_codex_trace_to_training_example(
     metadata = {
         "source_file": trace_file.name,
         "session_id": session_meta.get("id") or trace_file.stem,
-        "trace_type": session_meta.get("source") or "codex",
+        "trace_type": "codex",
+        "source": session_meta.get("source"),
         "model_provider": session_meta.get("model_provider"),
+        "model": (
+            session_meta.get("model")
+            or next(
+                (
+                    context.get("model")
+                    for context in reversed(turn_contexts)
+                    if isinstance(context.get("model"), str) and context.get("model")
+                ),
+                None,
+            )
+        ),
         "cwd": session_meta.get("cwd"),
         "cli_version": session_meta.get("cli_version"),
         "system_prompt": system_prompt or session_meta.get("system_prompt"),

src/teich/runner.py

@@ -2695,11 +2695,20 @@ def _build_external_docker_base_command(
         command.append(self.image_name)
         return command
 
-    def _build_shell_command(self, *, continue_session: bool = False) -> str:
+    def _build_shell_command(
+        self,
+        *,
+        continue_session: bool = False,
+        resume_session_id: str | None = None,
+    ) -> str:
         raise NotImplementedError
 
     def _wrap_external_shell_command(self, shell_command: str) -> str:
-        chmod_command = f"chmod -R a+rwX {shlex.quote(self.home_in_container)} >/dev/null 2>&1 || true"
+        chmod_targets = " ".join(
+            shlex.quote(path)
+            for path in (self.home_in_container, WORKSPACE_IN_CONTAINER)
+        )
+        chmod_command = f"chmod -R a+rwX {chmod_targets} >/dev/null 2>&1 || true"
         return f"set +e; {shell_command}; status=$?; {chmod_command}; exit $status"
 
     def _build_external_command(
@@ -2709,10 +2718,14 @@ def _build_external_command(
         container_name: str,
         *,
         continue_session: bool = False,
+        resume_session_id: str | None = None,
     ) -> list[str]:
         command = self._build_external_docker_base_command(workspace, home_dir, container_name)
         shell_command = self._wrap_external_shell_command(
-            self._build_shell_command(continue_session=continue_session)
+            self._build_shell_command(
+                continue_session=continue_session,
+                resume_session_id=resume_session_id,
+            )
         )
         command.extend(["bash", "-lc", shell_command])
         return command
@@ -2727,7 +2740,13 @@ def _build_external_persistent_container_command(
         command.extend(["sleep", "infinity"])
         return command
 
-    def _build_external_exec_command(self, container_name: str, *, continue_session: bool = False) -> list[str]:
+    def _build_external_exec_command(
+        self,
+        container_name: str,
+        *,
+        continue_session: bool = False,
+        resume_session_id: str | None = None,
+    ) -> list[str]:
         return [
             "docker",
             "exec",
@@ -2738,7 +2757,12 @@ def _build_external_exec_command(self, container_name: str, *, continue_session:
             container_name,
             "bash",
             "-lc",
-            self._wrap_external_shell_command(self._build_shell_command(continue_session=continue_session)),
+            self._wrap_external_shell_command(
+                self._build_shell_command(
+                    continue_session=continue_session,
+                    resume_session_id=resume_session_id,
+                )
+            ),
         ]
 
     def _run_external_process(self, command: list[str], container_name: str | None) -> tuple[str, str]:
@@ -3067,7 +3091,12 @@ def _openrouter_proxy_shell_prefix(self) -> str:
         )
         return f"node {shlex.quote(proxy_script)} >/tmp/claude-openrouter-proxy.log 2>&1 & {readiness_probe}"
 
-    def _build_shell_command(self, *, continue_session: bool = False) -> str:
+    def _build_shell_command(
+        self,
+        *,
+        continue_session: bool = False,
+        resume_session_id: str | None = None,
+    ) -> str:
         claude_command = [
             "claude",
             "-p",
@@ -3080,7 +3109,9 @@ def _build_shell_command(self, *, continue_session: bool = False) -> str:
         permission_mode = self._permission_mode()
         if permission_mode:
             claude_command.extend(["--permission-mode", permission_mode])
-        if continue_session:
+        if resume_session_id:
+            claude_command.extend(["--resume", resume_session_id])
+        elif continue_session:
             claude_command.append("--continue")
         return f"{shlex.join(claude_command)} < {shlex.quote(WORKSPACE_IN_CONTAINER + '/' + TEICH_PROMPT_FILE_NAME)}"
 
@@ -3091,18 +3122,21 @@ def _build_external_command(
         container_name: str,
         *,
         continue_session: bool = False,
+        resume_session_id: str | None = None,
     ) -> list[str]:
         if not self._needs_openrouter_model_proxy():
             return super()._build_external_command(
                 workspace,
                 home_dir,
                 container_name,
                 continue_session=continue_session,
+                resume_session_id=resume_session_id,
             )
         self._write_openrouter_proxy(home_dir)
         command = self._build_external_docker_base_command(workspace, home_dir, container_name)
         shell_command = self._wrap_external_shell_command(
-            f"{self._openrouter_proxy_shell_prefix()}{self._build_shell_command(continue_session=continue_session)}"
+            f"{self._openrouter_proxy_shell_prefix()}"
+            f"{self._build_shell_command(continue_session=continue_session, resume_session_id=resume_session_id)}"
         )
         command.extend(
             [
@@ -3126,6 +3160,17 @@ def _build_external_persistent_container_command(
         command.extend(["bash", "-lc", f"{self._openrouter_proxy_shell_prefix()}exec sleep infinity"])
         return command
 
+    @staticmethod
+    def _native_session_id_from_file(trace_path: Path) -> str | None:
+        events = _read_jsonl_dict_events(trace_path)
+        if not events:
+            return None
+        for event in events:
+            session_id = event.get("sessionId")
+            if isinstance(session_id, str) and session_id.strip():
+                return session_id.strip()
+        return None
+
     def _run_native_process_with_progress(
         self,
         command: list[str],
@@ -3258,6 +3303,7 @@ def run_session(
         failure_trace_saved = False
         failure_trace_checked = False
         existing_sessions: set[Path] = set()
+        native_resume_session_id: str | None = None
         try:
             workspace.mkdir(parents=True, exist_ok=True)
             existing_sessions = {path.resolve() for path in self._list_native_session_files(home_dir)}
@@ -3272,14 +3318,20 @@ def run_session(
                 expected_turns = turn_prompts[: turn_index + 1]
                 for attempt in range(AGENT_TURN_RETRY_LIMIT + 1):
                     continue_session = turn_index > 0 or attempt > 0
+                    resume_session_id = native_resume_session_id if continue_session else None
                     if len(turn_prompts) > 1:
-                        command = self._build_external_exec_command(container_name, continue_session=continue_session)
+                        command = self._build_external_exec_command(
+                            container_name,
+                            continue_session=continue_session,
+                            resume_session_id=resume_session_id,
+                        )
                     else:
                         command = self._build_external_command(
                             workspace,
                             home_dir,
                             container_name,
                             continue_session=continue_session,
+                            resume_session_id=resume_session_id,
                         )
                     process_error: subprocess.CalledProcessError | None = None
                     details = ""
@@ -3301,6 +3353,8 @@ def run_session(
                         details = stderr.strip() or stdout.strip() or str(exc)
 
                     source_trace = self._latest_native_session_file(home_dir, existing_sessions, started_at)
+                    if source_trace is not None:
+                        native_resume_session_id = self._native_session_id_from_file(source_trace) or native_resume_session_id
                     if source_trace is not None and self._trace_contains_completed_turns(source_trace, expected_turns):
                         break
                     if attempt < AGENT_TURN_RETRY_LIMIT:
@@ -3426,7 +3480,12 @@ def _hermes_cli_provider(self) -> str:
             return "custom"
         return self.config.api.provider
 
-    def _build_shell_command(self, *, continue_session: bool = False) -> str:
+    def _build_shell_command(
+        self,
+        *,
+        continue_session: bool = False,
+        resume_session_id: str | None = None,
+    ) -> str:
         prompt_path = shlex.quote(WORKSPACE_IN_CONTAINER + "/" + TEICH_PROMPT_FILE_NAME)
         hermes_command = [
             "hermes",

tests/test_converter.py

@@ -33,6 +33,7 @@ def test_detect_trace_type_returns_known_trace_type():
             ],
             "openclaw",
         ),
+        ([{"type": "external_session_meta", "payload": {"source": "hermes-agent"}}], "hermes"),
         ([{"type": "external_session_meta", "payload": {"source": "custom-agent"}}], "external_agent"),
     ]
 
@@ -377,9 +378,18 @@ def test_convert_codex_trace_keeps_system_prompt_and_tools_without_tool_calls(tm
             "type": "session_meta",
             "payload": {
                 "id": "codex-session-1",
+                "source": "exec",
+                "model_provider": "openrouter",
                 "base_instructions": {"text": "You are a careful coding agent."},
             },
         },
+        {
+            "type": "turn_context",
+            "payload": {
+                "turn_id": "turn-1",
+                "model": "google/gemini-3.1-flash-lite",
+            },
+        },
         {
             "type": "response_item",
             "timestamp": "2026-05-13T06:03:00.000Z",
@@ -423,6 +433,10 @@ def test_convert_codex_trace_keeps_system_prompt_and_tools_without_tool_calls(tm
     example = convert_trace_to_training_example(trace_file)
 
     assert example.messages[0] == {"role": "system", "content": "You are a careful coding agent."}
+    assert example.metadata["trace_type"] == "codex"
+    assert example.metadata["source"] == "exec"
+    assert example.metadata["model_provider"] == "openrouter"
+    assert example.metadata["model"] == "google/gemini-3.1-flash-lite"
     assert example.metadata["system_prompt"] == "You are a careful coding agent."
     assert example.metadata["first_message_timestamp"] == "2026-05-13T06:03:06.000Z"
     assert [tool["function"]["name"] for tool in example.tools] == ["exec_command"]
@@ -829,7 +843,7 @@ def test_convert_native_claude_code_orders_fragmented_assistant_turn_semanticall
     ]
 
 
-def test_convert_external_agent_trace(tmp_path: Path):
+def test_convert_teich_hermes_external_trace(tmp_path: Path):
     trace_file = tmp_path / "hermes.jsonl"
     events = [
         {
@@ -853,7 +867,8 @@ def test_convert_external_agent_trace(tmp_path: Path):
 
     example = convert_trace_to_training_example(trace_file)
 
-    assert example.metadata["trace_type"] == "hermes-agent"
+    assert example.metadata["trace_type"] == "hermes"
+    assert example.metadata["source"] == "hermes-agent"
     assert example.metadata["session_id"] == "hermes-session"
     assert example.metadata["first_message_timestamp"] == "2026-05-15T00:00:01.000Z"
     assert example.prompt == "Build a CLI"
@@ -863,7 +878,7 @@ def test_convert_external_agent_trace(tmp_path: Path):
     ]
 
 
-def test_convert_external_agent_trace_uses_meta_tools_without_tool_calls(tmp_path: Path):
+def test_convert_teich_hermes_external_trace_uses_meta_tools_without_tool_calls(tmp_path: Path):
     trace_file = tmp_path / "hermes-tools.jsonl"
     events = [
         {
@@ -896,13 +911,13 @@ def test_convert_external_agent_trace_uses_meta_tools_without_tool_calls(tmp_pat
 
     example = convert_trace_to_training_example(trace_file)
 
-    assert example.metadata["trace_type"] == "hermes-agent"
+    assert example.metadata["trace_type"] == "hermes"
     assert example.messages[-1] == {"role": "assistant", "content": "Done."}
     assert [tool["function"]["name"] for tool in example.tools] == ["delegate_task"]
     assert example.tools[0]["function"]["parameters"]["required"] == ["goal"]
 
 
-def test_convert_external_agent_trace_preserves_hermes_tool_calls_and_parent_metadata(tmp_path: Path):
+def test_convert_teich_hermes_external_trace_preserves_tool_calls_and_parent_metadata(tmp_path: Path):
     trace_file = tmp_path / "hermes-child.jsonl"
     events = [
         {
@@ -954,6 +969,7 @@ def test_convert_external_agent_trace_preserves_hermes_tool_calls_and_parent_met
 
     example = convert_trace_to_training_example(trace_file)
 
+    assert example.metadata["trace_type"] == "hermes"
     assert example.metadata["parent_session_id"] == "parent-session"
     assert example.metadata["tool_call_count"] == 1
     assert example.metadata["input_tokens"] == 12