diff --git a/crates/cli/tests/cli_tests.rs b/crates/cli/tests/cli_tests.rs index a3533022e..3a67b9d44 100644 --- a/crates/cli/tests/cli_tests.rs +++ b/crates/cli/tests/cli_tests.rs @@ -141,6 +141,33 @@ fn cli_easy_path_invokes_setup_when_no_config_found() { ); } +#[test] +fn cli_hermes_easy_path_invokes_setup_when_no_config_found() { + let temp = tempfile::tempdir().unwrap(); + let xdg = temp.path().join("xdg"); + std::fs::create_dir_all(&xdg).unwrap(); + let cwd = temp.path().join("workdir"); + std::fs::create_dir_all(&cwd).unwrap(); + + let output = Command::new(gateway_bin()) + .current_dir(&cwd) + .env("XDG_CONFIG_HOME", &xdg) + .env("HOME", temp.path()) + .arg("hermes") + .output() + .unwrap(); + + assert!( + !output.status.success(), + "Hermes easy path should exit non-zero when no config + no TTY for setup" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("setup requires a TTY"), + "expected non-TTY setup error in stderr, got:\n{stderr}" + ); +} + #[test] fn cli_bare_invocation_invokes_setup_when_no_config_found() { let temp = tempfile::tempdir().unwrap(); @@ -383,6 +410,39 @@ fn cli_hook_forward_posts_payload_headers_and_prints_response() { assert!(request.contains(r#"{"hook_event_name":"sessionStart"}"#)); } +#[test] +fn cli_hook_forward_hermes_shell_hook_returns_empty_object() { + let (server_url, received) = spawn_single_request_server(200, r#"{}"#); + let mut child = Command::new(gateway_bin()) + .args([ + "hook-forward", + "hermes", + "--gateway-url", + &server_url, + "--fail-closed", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(br#"{"session_id":"smoke-hermes","hook_event_name":"on_session_start"}"#) + .unwrap(); + let output = child.wait_with_output().unwrap(); + let request = received.recv().unwrap(); + + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), r#"{}"#); + assert!(request.contains("POST /hooks/hermes HTTP/1.1")); + assert!( + request.contains(r#"{"session_id":"smoke-hermes","hook_event_name":"on_session_start"}"#) + ); +} + #[test] fn cli_hook_forward_reports_http_failure_when_fail_closed() { let (server_url, received) = spawn_single_request_server(503, "unavailable"); diff --git a/crates/cli/tests/coverage/server_tests.rs b/crates/cli/tests/coverage/server_tests.rs index 1d6ac2ad8..677a38e79 100644 --- a/crates/cli/tests/coverage/server_tests.rs +++ b/crates/cli/tests/coverage/server_tests.rs @@ -321,6 +321,206 @@ async fn serve_listener_observability_plugin_records_non_hermes_hooks() { assert!(!agent_starts.contains(&"claude-code".to_string())); } +#[tokio::test] +async fn serve_listener_hermes_api_hooks_write_atof_category_profile_and_fidelity() { + let _guard = PLUGIN_TEST_LOCK.lock().await; + let _ = nemo_relay::plugin::clear_plugin_configuration(); + + let temp = tempfile::tempdir().unwrap(); + let atof_dir = temp.path().join("atof"); + std::fs::create_dir_all(&atof_dir).unwrap(); + let mut config = test_config(); + config.plugin_config = Some(json!({ + "version": 1, + "components": [ + { + "kind": "observability", + "enabled": true, + "config": { + "version": 1, + "atof": { + "enabled": true, + "output_directory": atof_dir, + "filename": "events.jsonl", + "mode": "overwrite" + } + } + } + ] + })); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let url = format!("http://{address}"); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let handle = + tokio::spawn(async move { serve_listener(listener, config, Some(shutdown_rx)).await }); + + wait_for_gateway(&url).await; + let client = test_http_client(); + + let response = client + .post(format!("{url}/hooks/hermes")) + .json(&json!({ + "hook_event_name": "pre_api_request", + "session_id": "hermes-atof-exact", + "extra": { + "task_id": "task-1", + "api_request_id": "turn-1:api:2", + "api_call_count": 2, + "model": "qwen", + "provider": "custom", + "request": { + "method": "POST", + "body": { + "model": "qwen", + "messages": [ + { "role": "user", "content": "hello" } + ], + "tools": [ + { "type": "function", "function": { "name": "search_files" } } + ] + } + } + } + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let response = client + .post(format!("{url}/hooks/hermes")) + .json(&json!({ + "hook_event_name": "post_api_request", + "session_id": "hermes-atof-exact", + "extra": { + "task_id": "task-1", + "api_request_id": "turn-1:api:2", + "api_call_count": 2, + "model": "qwen", + "response": { + "model": "qwen", + "finish_reason": "tool_calls", + "assistant_message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "search_files", + "arguments": "{\"query\":\"needle\"}" + } + } + ] + }, + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "cost": { "total": 0.0042 } + } + } + } + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let response = client + .post(format!("{url}/hooks/hermes")) + .json(&json!({ + "hook_event_name": "pre_api_request", + "session_id": "hermes-atof-lossy", + "extra": { + "task_id": "task-2", + "api_call_count": 4, + "model": "qwen", + "provider": "custom", + "request": null, + "message_count": 2 + } + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + shutdown_tx.send(()).unwrap(); + handle.await.unwrap().unwrap(); + + let events = std::fs::read_to_string(temp.path().join("atof/events.jsonl")).unwrap(); + let llm_events = events + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|event| event["category"] == "llm") + .collect::>(); + assert_eq!( + llm_events.len(), + 4, + "expected Hermes LLM exports, got {llm_events:?}" + ); + + let start = llm_events + .iter() + .find(|event| { + event["scope_category"] == "start" + && event["metadata"]["api_call_id"] == json!("turn-1:api:2") + }) + .unwrap(); + assert_eq!(start["category_profile"]["model_name"], json!("qwen")); + assert_eq!(start["metadata"]["provider_payload_exact"], json!(true)); + assert_eq!( + start["metadata"]["fidelity_source"], + json!("hermes_api_hooks_sanitized") + ); + assert_eq!( + start["data"]["content"]["messages"][0]["content"], + json!("hello") + ); + assert_eq!( + start["data"]["content"]["tools"][0]["function"]["name"], + json!("search_files") + ); + + let end = llm_events + .iter() + .find(|event| { + event["scope_category"] == "end" + && event["metadata"]["api_call_id"] == json!("turn-1:api:2") + }) + .unwrap(); + assert_eq!(end["category_profile"]["model_name"], json!("qwen")); + assert_eq!(end["metadata"]["provider_payload_exact"], json!(true)); + assert_eq!(end["data"]["tool_calls"][0]["id"], json!("call-1")); + assert_eq!( + end["data"]["tool_calls"][0]["function"]["name"], + json!("search_files") + ); + assert_eq!(end["data"]["usage"]["prompt_tokens"], json!(10)); + assert_eq!(end["data"]["usage"]["completion_tokens"], json!(5)); + + let lossy_start = llm_events + .iter() + .find(|event| { + event["scope_category"] == "start" + && event["metadata"]["api_call_id"] == json!("hermes-atof-lossy:task-2:4") + }) + .unwrap(); + assert_eq!(lossy_start["category_profile"]["model_name"], json!("qwen")); + assert_eq!( + lossy_start["metadata"]["provider_payload_exact"], + json!(false) + ); + assert_eq!( + lossy_start["data"]["content"]["fidelity"]["provider_payload_exact"], + json!(false) + ); + assert_eq!(lossy_start["data"]["content"]["message_count"], json!(2)); +} + #[tokio::test] async fn serve_listener_activates_any_registered_plugin_kind() { let _guard = PLUGIN_TEST_LOCK.lock().await; diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index fc8d11f6f..a8910b736 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -2136,6 +2136,105 @@ fn annotated_llm_payloads_emit_flattened_openinference_message_and_tool_attribut assert_attr(&attributes, "llm.finish_reason", "tool_use"); } +#[test] +fn hermes_exact_api_payloads_emit_openinference_text_usage_and_metadata() { + let (provider, exporter) = make_provider(); + let mut processor = + OpenInferenceEventProcessor::new(provider.clone(), "test-scope".to_string()); + let uuid = Uuid::now_v7(); + let metadata = json!({ + "provider_payload_exact": true, + "fidelity_source": "hermes_api_hooks_sanitized" + }); + + processor.process(&Event::Scope(ScopeEvent::new( + BaseEvent::builder() + .uuid(uuid) + .name("custom") + .data(json!({ + "model": "qwen", + "messages": [{ "role": "user", "content": "hello" }], + "tools": [ + { "type": "function", "function": { "name": "search_files" } } + ] + })) + .metadata(metadata.clone()) + .build(), + ScopeCategory::Start, + Vec::new(), + EventCategory::llm(), + Some(CategoryProfile::builder().model_name("qwen").build()), + ))); + processor.process(&Event::Scope(ScopeEvent::new( + BaseEvent::builder() + .uuid(uuid) + .name("custom") + .data(json!({ + "content": "", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "search_files", + "arguments": "{\"query\":\"needle\"}" + } + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "cost": { "total": 0.0042 } + }, + "model": "qwen", + "finish_reason": "tool_calls" + })) + .metadata(metadata) + .build(), + ScopeCategory::End, + Vec::new(), + EventCategory::llm(), + Some(CategoryProfile::builder().model_name("qwen").build()), + ))); + + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1); + let attributes = attr_map(&spans[0].attributes); + assert_eq!( + attributes.get("openinference.span.kind"), + Some(&"LLM".to_string()) + ); + assert_eq!(attributes.get("llm.model_name"), Some(&"qwen".to_string())); + assert_eq!( + attributes.get("input.value"), + Some(&"user: hello".to_string()) + ); + assert_eq!( + attributes.get("output.value"), + Some(&"Requested tools: search_files".to_string()) + ); + assert_eq!( + attributes.get("llm.token_count.prompt"), + Some(&"10".to_string()) + ); + assert_eq!( + attributes.get("llm.token_count.completion"), + Some(&"5".to_string()) + ); + assert_eq!( + attributes.get("llm.cost.total"), + Some(&"0.0042".to_string()) + ); + assert_attr_contains(&attributes, "metadata", "\"provider_payload_exact\":true"); + assert_attr_contains( + &attributes, + "metadata", + "\"fidelity_source\":\"hermes_api_hooks_sanitized\"", + ); +} + #[test] fn llm_end_with_inconsistent_manual_usage_omits_invalid_total_tokens() { let (provider, exporter) = make_provider(); diff --git a/docs/nemo-relay-cli/hermes.mdx b/docs/nemo-relay-cli/hermes.mdx index 9656d0a15..ad4e4c433 100644 --- a/docs/nemo-relay-cli/hermes.mdx +++ b/docs/nemo-relay-cli/hermes.mdx @@ -8,14 +8,14 @@ SPDX-License-Identifier: Apache-2.0 */} Use this guide to observe local Hermes Agent sessions with NeMo Relay through -Hermes shell hooks and the `nemo-relay` gateway. This gateway path is -separate from the Hermes third-party patch set under `patches/hermes-agent/`; -use the gateway when you want hook forwarding without rebuilding a patched -Hermes checkout. +Hermes shell hooks and the `nemo-relay` gateway. -Hermes shell hooks provide session, subagent, tool, and LLM hint lifecycle -events. Complete LLM request and response observability still requires model -traffic to route through the gateway. +Hermes shell hooks provide session, subagent, tool, and LLM lifecycle events. +For NeMo Relay contract work, `pre_api_request`, `post_api_request`, and +`api_request_error` are the authoritative LLM lifecycle hooks. The legacy +`pre_llm_call` and `post_llm_call` hooks still exist, but only as private +hint-style signals. Complete LLM request and response observability still +requires model traffic to route through the gateway. ## Transparent Run @@ -32,11 +32,18 @@ Pass Hermes arguments after `--`: nemo-relay hermes -- chat --provider custom ``` -This shortcut is equivalent to `nemo-relay run -- hermes`. The wrapper starts a -gateway on a dynamic `127.0.0.1` port and exports `NEMO_RELAY_GATEWAY_URL` for -the launched process. Hermes hook configuration is not temporary in this mode. -Install hooks first, or configure equivalent Hermes shell hooks, so approved -hook commands can discover the dynamic gateway URL. +Once NeMo Relay config exists, this shortcut is equivalent to +`nemo-relay run --agent hermes`. The wrapper starts a gateway on a dynamic +`127.0.0.1` port and exports `NEMO_RELAY_GATEWAY_URL` for the launched +process. After initial NeMo Relay setup exists, Hermes hook configuration is +temporary in this mode: the launcher merges the NeMo Relay hook-forward +commands into the configured Hermes hook file for the run and restores the +original file afterward. The wrapper also sets `HERMES_ACCEPT_HOOKS=1` so +Hermes can use the injected hook commands without extra manual approval +prompts. + +If no NeMo Relay config exists yet, `nemo-relay hermes` triggers the setup flow +first and then launches Hermes through the same wrapped pipeline. Inspect what would be launched without starting Hermes: @@ -89,7 +96,13 @@ records the path under `[agents.hermes].hooks_path` in `.nemo-relay/config.toml` The generated Hermes hooks cover `on_session_start`, `on_session_end`, `on_session_finalize`, `on_session_reset`, `pre_llm_call`, `post_llm_call`, -`pre_tool_call`, `post_tool_call`, `subagent_start`, and `subagent_stop`. +`pre_api_request`, `post_api_request`, `api_request_error`, `pre_tool_call`, +`post_tool_call`, `subagent_start`, and `subagent_stop`. + +The API hooks are the main Hermes LLM lifecycle path for NeMo Relay. The legacy +LLM hooks remain installed because they can still provide useful private hints, +but they are not treated as equal peers to the API hooks in the observability +contract. Hermes hook forwarding prefers `NEMO_RELAY_GATEWAY_URL` when set (this is what `nemo-relay hermes` injects on every run). When launched outside the wrapper — @@ -104,6 +117,16 @@ nemo-relay --bind 127.0.0.1:4040 Then point Hermes provider traffic at `http://127.0.0.1:4040` for any provider mode that exposes a local OpenAI-compatible or Anthropic-compatible base URL. +This is the practical routed-provider validation path today. + +Important distinction: + +- Wrapped execution is the authoritative path for Hermes hook-path validation. +- Wrapped execution does not automatically rewrite Hermes provider `base_url`, + `custom_providers`, or `api_mode`. +- Routed `/v1/messages`, `/v1/chat/completions`, or `/v1/responses` validation + therefore requires explicit Hermes provider configuration in addition to the + wrapped or standalone gateway. ## Smoke Test