test: tighten Hermes wrapped observability contract

crates/cli/tests/cli_tests.rs

@@ -141,6 +141,33 @@ fn cli_easy_path_invokes_setup_when_no_config_found() {
     );
 }
 
+#[test]
+fn cli_hermes_easy_path_invokes_setup_when_no_config_found() {
+    let temp = tempfile::tempdir().unwrap();
+    let xdg = temp.path().join("xdg");
+    std::fs::create_dir_all(&xdg).unwrap();
+    let cwd = temp.path().join("workdir");
+    std::fs::create_dir_all(&cwd).unwrap();
+
+    let output = Command::new(gateway_bin())
+        .current_dir(&cwd)
+        .env("XDG_CONFIG_HOME", &xdg)
+        .env("HOME", temp.path())
+        .arg("hermes")
+        .output()
+        .unwrap();
+
+    assert!(
+        !output.status.success(),
+        "Hermes easy path should exit non-zero when no config + no TTY for setup"
+    );
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    assert!(
+        stderr.contains("setup requires a TTY"),
+        "expected non-TTY setup error in stderr, got:\n{stderr}"
+    );
+}
+
 #[test]
 fn cli_bare_invocation_invokes_setup_when_no_config_found() {
     let temp = tempfile::tempdir().unwrap();
@@ -383,6 +410,39 @@ fn cli_hook_forward_posts_payload_headers_and_prints_response() {
     assert!(request.contains(r#"{"hook_event_name":"sessionStart"}"#));
 }
 
+#[test]
+fn cli_hook_forward_hermes_shell_hook_returns_empty_object() {
+    let (server_url, received) = spawn_single_request_server(200, r#"{}"#);
+    let mut child = Command::new(gateway_bin())
+        .args([
+            "hook-forward",
+            "hermes",
+            "--gateway-url",
+            &server_url,
+            "--fail-closed",
+        ])
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .unwrap();
+    child
+        .stdin
+        .take()
+        .unwrap()
+        .write_all(br#"{"session_id":"smoke-hermes","hook_event_name":"on_session_start"}"#)
+        .unwrap();
+    let output = child.wait_with_output().unwrap();
+    let request = received.recv().unwrap();
+
+    assert!(output.status.success());
+    assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), r#"{}"#);
+    assert!(request.contains("POST /hooks/hermes HTTP/1.1"));
+    assert!(
+        request.contains(r#"{"session_id":"smoke-hermes","hook_event_name":"on_session_start"}"#)
+    );
+}
+
 #[test]
 fn cli_hook_forward_reports_http_failure_when_fail_closed() {
     let (server_url, received) = spawn_single_request_server(503, "unavailable");

crates/cli/tests/coverage/server_tests.rs

@@ -321,6 +321,206 @@ async fn serve_listener_observability_plugin_records_non_hermes_hooks() {
     assert!(!agent_starts.contains(&"claude-code".to_string()));
 }
 
+#[tokio::test]
+async fn serve_listener_hermes_api_hooks_write_atof_category_profile_and_fidelity() {
+    let _guard = PLUGIN_TEST_LOCK.lock().await;
+    let _ = nemo_relay::plugin::clear_plugin_configuration();
+
+    let temp = tempfile::tempdir().unwrap();
+    let atof_dir = temp.path().join("atof");
+    std::fs::create_dir_all(&atof_dir).unwrap();
+    let mut config = test_config();
+    config.plugin_config = Some(json!({
+        "version": 1,
+        "components": [
+            {
+                "kind": "observability",
+                "enabled": true,
+                "config": {
+                    "version": 1,
+                    "atof": {
+                        "enabled": true,
+                        "output_directory": atof_dir,
+                        "filename": "events.jsonl",
+                        "mode": "overwrite"
+                    }
+                }
+            }
+        ]
+    }));
+
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    let url = format!("http://{address}");
+    let (shutdown_tx, shutdown_rx) = oneshot::channel();
+    let handle =
+        tokio::spawn(async move { serve_listener(listener, config, Some(shutdown_rx)).await });
+
+    wait_for_gateway(&url).await;
+    let client = test_http_client();
+
+    let response = client
+        .post(format!("{url}/hooks/hermes"))
+        .json(&json!({
+            "hook_event_name": "pre_api_request",
+            "session_id": "hermes-atof-exact",
+            "extra": {
+                "task_id": "task-1",
+                "api_request_id": "turn-1:api:2",
+                "api_call_count": 2,
+                "model": "qwen",
+                "provider": "custom",
+                "request": {
+                    "method": "POST",
+                    "body": {
+                        "model": "qwen",
+                        "messages": [
+                            { "role": "user", "content": "hello" }
+                        ],
+                        "tools": [
+                            { "type": "function", "function": { "name": "search_files" } }
+                        ]
+                    }
+                }
+            }
+        }))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let response = client
+        .post(format!("{url}/hooks/hermes"))
+        .json(&json!({
+            "hook_event_name": "post_api_request",
+            "session_id": "hermes-atof-exact",
+            "extra": {
+                "task_id": "task-1",
+                "api_request_id": "turn-1:api:2",
+                "api_call_count": 2,
+                "model": "qwen",
+                "response": {
+                    "model": "qwen",
+                    "finish_reason": "tool_calls",
+                    "assistant_message": {
+                        "role": "assistant",
+                        "content": "",
+                        "tool_calls": [
+                            {
+                                "id": "call-1",
+                                "type": "function",
+                                "function": {
+                                    "name": "search_files",
+                                    "arguments": "{\"query\":\"needle\"}"
+                                }
+                            }
+                        ]
+                    },
+                    "usage": {
+                        "prompt_tokens": 10,
+                        "completion_tokens": 5,
+                        "cost": { "total": 0.0042 }
+                    }
+                }
+            }
+        }))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let response = client
+        .post(format!("{url}/hooks/hermes"))
+        .json(&json!({
+            "hook_event_name": "pre_api_request",
+            "session_id": "hermes-atof-lossy",
+            "extra": {
+                "task_id": "task-2",
+                "api_call_count": 4,
+                "model": "qwen",
+                "provider": "custom",
+                "request": null,
+                "message_count": 2
+            }
+        }))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::OK);
+
+    shutdown_tx.send(()).unwrap();
+    handle.await.unwrap().unwrap();
+
+    let events = std::fs::read_to_string(temp.path().join("atof/events.jsonl")).unwrap();
+    let llm_events = events
+        .lines()
+        .map(|line| serde_json::from_str::<Value>(line).unwrap())
+        .filter(|event| event["category"] == "llm")
+        .collect::<Vec<_>>();
+    assert_eq!(
+        llm_events.len(),
+        4,
+        "expected Hermes LLM exports, got {llm_events:?}"
+    );
+
+    let start = llm_events
+        .iter()
+        .find(|event| {
+            event["scope_category"] == "start"
+                && event["metadata"]["api_call_id"] == json!("turn-1:api:2")
+        })
+        .unwrap();
+    assert_eq!(start["category_profile"]["model_name"], json!("qwen"));
+    assert_eq!(start["metadata"]["provider_payload_exact"], json!(true));
+    assert_eq!(
+        start["metadata"]["fidelity_source"],
+        json!("hermes_api_hooks_sanitized")
+    );
+    assert_eq!(
+        start["data"]["content"]["messages"][0]["content"],
+        json!("hello")
+    );
+    assert_eq!(
+        start["data"]["content"]["tools"][0]["function"]["name"],
+        json!("search_files")
+    );
+
+    let end = llm_events
+        .iter()
+        .find(|event| {
+            event["scope_category"] == "end"
+                && event["metadata"]["api_call_id"] == json!("turn-1:api:2")
+        })
+        .unwrap();
+    assert_eq!(end["category_profile"]["model_name"], json!("qwen"));
+    assert_eq!(end["metadata"]["provider_payload_exact"], json!(true));
+    assert_eq!(end["data"]["tool_calls"][0]["id"], json!("call-1"));
+    assert_eq!(
+        end["data"]["tool_calls"][0]["function"]["name"],
+        json!("search_files")
+    );
+    assert_eq!(end["data"]["usage"]["prompt_tokens"], json!(10));
+    assert_eq!(end["data"]["usage"]["completion_tokens"], json!(5));
+
+    let lossy_start = llm_events
+        .iter()
+        .find(|event| {
+            event["scope_category"] == "start"
+                && event["metadata"]["api_call_id"] == json!("hermes-atof-lossy:task-2:4")
+        })
+        .unwrap();
+    assert_eq!(lossy_start["category_profile"]["model_name"], json!("qwen"));
+    assert_eq!(
+        lossy_start["metadata"]["provider_payload_exact"],
+        json!(false)
+    );
+    assert_eq!(
+        lossy_start["data"]["content"]["fidelity"]["provider_payload_exact"],
+        json!(false)
+    );
+    assert_eq!(lossy_start["data"]["content"]["message_count"], json!(2));
+}
+
 #[tokio::test]
 async fn serve_listener_activates_any_registered_plugin_kind() {
     let _guard = PLUGIN_TEST_LOCK.lock().await;

crates/core/tests/unit/observability/openinference_tests.rs

@@ -2136,6 +2136,105 @@ fn annotated_llm_payloads_emit_flattened_openinference_message_and_tool_attribut
     assert_attr(&attributes, "llm.finish_reason", "tool_use");
 }
 
+#[test]
+fn hermes_exact_api_payloads_emit_openinference_text_usage_and_metadata() {
+    let (provider, exporter) = make_provider();
+    let mut processor =
+        OpenInferenceEventProcessor::new(provider.clone(), "test-scope".to_string());
+    let uuid = Uuid::now_v7();
+    let metadata = json!({
+        "provider_payload_exact": true,
+        "fidelity_source": "hermes_api_hooks_sanitized"
+    });
+
+    processor.process(&Event::Scope(ScopeEvent::new(
+        BaseEvent::builder()
+            .uuid(uuid)
+            .name("custom")
+            .data(json!({
+                "model": "qwen",
+                "messages": [{ "role": "user", "content": "hello" }],
+                "tools": [
+                    { "type": "function", "function": { "name": "search_files" } }
+                ]
+            }))
+            .metadata(metadata.clone())
+            .build(),
+        ScopeCategory::Start,
+        Vec::new(),
+        EventCategory::llm(),
+        Some(CategoryProfile::builder().model_name("qwen").build()),
+    )));
+    processor.process(&Event::Scope(ScopeEvent::new(
+        BaseEvent::builder()
+            .uuid(uuid)
+            .name("custom")
+            .data(json!({
+                "content": "",
+                "tool_calls": [
+                    {
+                        "id": "call-1",
+                        "type": "function",
+                        "function": {
+                            "name": "search_files",
+                            "arguments": "{\"query\":\"needle\"}"
+                        }
+                    }
+                ],
+                "usage": {
+                    "prompt_tokens": 10,
+                    "completion_tokens": 5,
+                    "cost": { "total": 0.0042 }
+                },
+                "model": "qwen",
+                "finish_reason": "tool_calls"
+            }))
+            .metadata(metadata)
+            .build(),
+        ScopeCategory::End,
+        Vec::new(),
+        EventCategory::llm(),
+        Some(CategoryProfile::builder().model_name("qwen").build()),
+    )));
+
+    processor.force_flush().unwrap();
+
+    let spans = exporter.get_finished_spans().unwrap();
+    assert_eq!(spans.len(), 1);
+    let attributes = attr_map(&spans[0].attributes);
+    assert_eq!(
+        attributes.get("openinference.span.kind"),
+        Some(&"LLM".to_string())
+    );
+    assert_eq!(attributes.get("llm.model_name"), Some(&"qwen".to_string()));
+    assert_eq!(
+        attributes.get("input.value"),
+        Some(&"user: hello".to_string())
+    );
+    assert_eq!(
+        attributes.get("output.value"),
+        Some(&"Requested tools: search_files".to_string())
+    );
+    assert_eq!(
+        attributes.get("llm.token_count.prompt"),
+        Some(&"10".to_string())
+    );
+    assert_eq!(
+        attributes.get("llm.token_count.completion"),
+        Some(&"5".to_string())
+    );
+    assert_eq!(
+        attributes.get("llm.cost.total"),
+        Some(&"0.0042".to_string())
+    );
+    assert_attr_contains(&attributes, "metadata", "\"provider_payload_exact\":true");
+    assert_attr_contains(
+        &attributes,
+        "metadata",
+        "\"fidelity_source\":\"hermes_api_hooks_sanitized\"",
+    );
+}
+
 #[test]
 fn llm_end_with_inconsistent_manual_usage_omits_invalid_total_tokens() {
     let (provider, exporter) = make_provider();