feat(response-cache): opt-in exact-match LLM response cache

An opt-in feature of the adaptive plugin (a response_cache config
section, not a new plugin kind): managed LLM calls are keyed by a
SHA-256 fingerprint of the normalized request and repeats are served
from the store — instant, free, reproducible. Buffered and streaming
calls share one keyspace; a streamed miss is teed and stored as its
aggregated response, a hit replays provider-native chunks.

Keying auto-detects the provider surface from the request shape and
trusts the decode only where it is faithful: known-lossy shapes and
decodes that fail to round-trip fall back to raw-body fingerprinting,
which can only cost a miss. Stateful calls (Responses persistence,
conversations, containers) and nondeterministic calls under the safety
toggle bypass entirely. Stored answers must be complete: non-null
errors, non-final statuses, truncated or lossily-collected streams, and
upstream failures are never cached — the CLI gateway relays failed
upstream replies to the client verbatim while keeping them invisible to
the execution chain.

The in-memory store is bounded by an honest resident-size budget with
oldest-first eviction; the Redis backend runs under hard deadlines and
re-checks entry expiry. Everything fails open: any cache error falls
through to a live call. Same config surface in Rust, Python, Node, and
Go; doctor reports configuration and backend reachability; hit/miss/
bypass marks carry fingerprints and savings, never bodies.

Signed-off-by: Zhongxuan Wang <daniewang@nvidia.com>

Author: zhongxuanwang-nv

.gitignore

@@ -44,6 +44,8 @@ integrations/openclaw/.test-dist/
 go/nemo_relay/coverage.out
 go_junit_report.xml
 *.dSYM/
+# Runtime observability capture files (written by examples/tests)
+nemo-relay-events-*.jsonl
 
 # OS — macOS
 .DS_Store
@@ -85,3 +87,12 @@ CHANGELOG.md
 
 # Relay
 /.nemo-relay
+
+# Local response-cache scratch files (proposals/, demos, benchmarks, quickstarts),
+# kept on disk for local use only, never committed.
+/proposals/
+crates/adaptive/examples/response_cache_*.rs
+python/examples/response_cache_*.py
+
+# Local-only response-cache notes (kept local, never committed)
+RESPONSE_CACHE*.md

crates/adaptive/Cargo.toml

@@ -22,6 +22,7 @@ chrono = { version = "0.4", features = ["serde"] }
 thiserror = "2"
 tdigest = { version = "0.2", features = ["use_serde"] }
 tokio = { version = "1", features = ["rt", "macros", "sync", "time"] }
+tokio-stream = { version = "0.1", default-features = false, features = ["sync"] }
 regex = "1"
 serde_json_canonicalizer = "0.3"
 sha2 = "0.11"
@@ -33,7 +34,6 @@ redis-backend = ["redis"]
 
 [dev-dependencies]
 tokio = { version = "1", default-features = false, features = ["rt", "macros", "sync", "time", "test-util", "rt-multi-thread"] }
-tokio-stream = { version = "0.1", default-features = false }
 
 [[test]]
 name = "redis_integration"
@@ -46,3 +46,11 @@ path = "tests/integration/runtime_integration_tests.rs"
 [[test]]
 name = "acg_module_surface"
 path = "tests/integration/acg_module_surface_tests.rs"
+
+[[test]]
+name = "response_cache_integration"
+path = "tests/integration/response_cache_tests.rs"
+
+[[test]]
+name = "response_cache_benchmark_integration"
+path = "tests/integration/response_cache_benchmark_tests.rs"

crates/adaptive/src/config.rs

@@ -7,6 +7,8 @@ use nemo_relay::plugin::ConfigPolicy;
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value as Json};
 
+use crate::response_cache::config::{BackendConfig, KEY_STRATEGY_EXACT_REQUEST};
+
 /// Canonical config document for the adaptive plugin component.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AdaptiveConfig {
@@ -32,6 +34,10 @@ pub struct AdaptiveConfig {
     /// Adaptive Cache Governor settings.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub acg: Option<AcgComponentConfig>,
+    /// Opt-in LLM response cache (exact-match). When present, the
+    /// adaptive plugin installs the response-cache execution intercept(s).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub response_cache: Option<ResponseCacheConfig>,
     /// Adaptive-local unsupported-config policy.
     #[serde(default)]
     pub policy: ConfigPolicy,
@@ -47,6 +53,7 @@ impl Default for AdaptiveConfig {
             adaptive_hints: None,
             tool_parallelism: None,
             acg: None,
+            response_cache: None,
             policy: ConfigPolicy::default(),
         }
     }
@@ -184,6 +191,55 @@ impl Default for AcgComponentConfig {
     }
 }
 
+/// Configuration for the adaptive plugin's `response_cache` feature
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+pub struct ResponseCacheConfig {
+    /// How long a stored answer stays reusable, in seconds.
+    pub ttl_seconds: u64,
+    /// Namespace folded into every key to separate environments/tenants.
+    pub namespace: String,
+    /// Execution-intercept priority; lower runs first/outermost (default `50`).
+    pub priority: i32,
+    /// Probability in `[0.0, 1.0]` of skipping the cache and running live.
+    pub bypass_rate: f64,
+    /// Cache nondeterministic requests too. Set `false` to cache only
+    /// requests explicitly pinned deterministic (`temperature` = 0) — absent
+    /// or unreadable temperatures count as nondeterministic.
+    pub cache_nondeterministic: bool,
+    /// Key strategy. Only [`KEY_STRATEGY_EXACT_REQUEST`] is supported.
+    pub key_strategy: String,
+    /// Request headers (case-insensitive) folded into the key; never auth headers.
+    pub header_allowlist: Vec<String>,
+    /// Extra top-level request-body keys to drop from the key, beyond the noise defaults.
+    pub skip_keys: Vec<String>,
+    /// Storage backend selection.
+    pub backend: BackendConfig,
+}
+
+impl Default for ResponseCacheConfig {
+    fn default() -> Self {
+        Self {
+            ttl_seconds: 3600,
+            namespace: String::new(),
+            priority: 50,
+            bypass_rate: 0.0,
+            cache_nondeterministic: true,
+            key_strategy: KEY_STRATEGY_EXACT_REQUEST.to_string(),
+            header_allowlist: Vec::new(),
+            skip_keys: Vec::new(),
+            backend: BackendConfig::default(),
+        }
+    }
+}
+
+impl ResponseCacheConfig {
+    /// TTL as a [`std::time::Duration`].
+    pub fn ttl(&self) -> std::time::Duration {
+        std::time::Duration::from_secs(self.ttl_seconds)
+    }
+}
+
 fn default_adaptive_config_version() -> u32 {
     1
 }
@@ -254,6 +310,13 @@ nemo_relay::editor_config! {
             nested: AcgComponentConfig,
             default: AcgComponentConfig,
         },
+        response_cache => {
+            label: "response_cache",
+            kind: Section,
+            optional: true,
+            nested: ResponseCacheConfig,
+            default: ResponseCacheConfig,
+        },
         policy => {
             label: "policy",
             kind: Section,
@@ -326,6 +389,25 @@ nemo_relay::editor_config! {
     }
 }
 
+nemo_relay::editor_config! {
+    impl ResponseCacheConfig {
+        ttl_seconds => { label: "ttl_seconds", kind: Integer },
+        namespace => { label: "namespace", kind: String },
+        priority => { label: "priority", kind: Integer },
+        bypass_rate => { label: "bypass_rate", kind: Float },
+        cache_nondeterministic => { label: "cache_nondeterministic", kind: Boolean },
+        key_strategy => { label: "key_strategy", kind: String },
+        header_allowlist => { label: "header_allowlist", kind: Json },
+        skip_keys => { label: "skip_keys", kind: Json },
+        backend => {
+            label: "backend",
+            kind: Section,
+            nested: BackendConfig,
+            default: BackendConfig,
+        },
+    }
+}
+
 nemo_relay::editor_config! {
     impl crate::acg::stability::StabilityThresholds {
         stable_threshold => { label: "stable_threshold", kind: Float },

crates/adaptive/src/lib.rs

@@ -30,6 +30,8 @@ pub mod learner;
 pub mod plugin_component;
 #[cfg(feature = "redis-backend")]
 pub mod redis;
+/// Opt-in LLM response cache (exact-match).
+pub mod response_cache;
 mod runtime;
 /// Storage backends and backend traits for adaptive state persistence.
 pub mod storage;
@@ -41,8 +43,8 @@ pub mod trie;
 pub mod types;
 
 pub use config::{
-    AcgComponentConfig, AdaptiveConfig, AdaptiveHintsComponentConfig, BackendSpec, StateConfig,
-    TelemetryComponentConfig, ToolParallelismComponentConfig,
+    AcgComponentConfig, AdaptiveConfig, AdaptiveHintsComponentConfig, BackendSpec,
+    ResponseCacheConfig, StateConfig, TelemetryComponentConfig, ToolParallelismComponentConfig,
 };
 pub use context_helpers::{
     LATENCY_SENSITIVITY_POINTER, extract_scope_path, read_manual_latency_sensitivity,
@@ -51,6 +53,7 @@ pub use context_helpers::{
 pub use error::{AdaptiveError, Result};
 #[cfg(feature = "redis-backend")]
 pub use redis::RedisBackend;
+pub use response_cache::RESPONSE_CACHE_MARK;
 pub use runtime::features::AdaptiveRuntime;
 pub use storage::erased::AnyBackend;
 pub use storage::memory::InMemoryBackend;

crates/adaptive/src/plugin_component.rs

@@ -182,6 +182,7 @@ fn validate_adaptive_plugin_config(plugin_config: &Map<String, Json>) -> Vec<Con
             "adaptive_hints",
             "tool_parallelism",
             "acg",
+            "response_cache",
             "policy",
         ],
     );
@@ -284,10 +285,75 @@ fn validate_adaptive_plugin_config(plugin_config: &Map<String, Json>) -> Vec<Con
         );
     }
 
+    if let Some(response_cache_json) = plugin_config
+        .get("response_cache")
+        .and_then(Json::as_object)
+    {
+        validate_unknown_fields(
+            &mut diagnostics,
+            &config.policy,
+            Some("response_cache".to_string()),
+            response_cache_json,
+            &[
+                "ttl_seconds",
+                "namespace",
+                "priority",
+                "bypass_rate",
+                "cache_nondeterministic",
+                "key_strategy",
+                "header_allowlist",
+                "skip_keys",
+                "backend",
+            ],
+        );
+        if let Some(backend_json) = response_cache_json.get("backend").and_then(Json::as_object) {
+            validate_unknown_fields(
+                &mut diagnostics,
+                &config.policy,
+                Some("response_cache.backend".to_string()),
+                backend_json,
+                &["kind", "config"],
+            );
+            let backend_kind = backend_json
+                .get("kind")
+                .and_then(Json::as_str)
+                .unwrap_or_default();
+            if let Some(backend_config_json) = backend_json.get("config").and_then(Json::as_object)
+            {
+                validate_response_cache_backend_config_fields(
+                    &mut diagnostics,
+                    &config.policy,
+                    backend_kind,
+                    backend_config_json,
+                );
+            }
+        }
+    }
+
     diagnostics.extend(AdaptiveRuntime::validate_config(&config).diagnostics);
     diagnostics
 }
 
+fn validate_response_cache_backend_config_fields(
+    diagnostics: &mut Vec<ConfigDiagnostic>,
+    policy: &ConfigPolicy,
+    backend_kind: &str,
+    backend_config: &Map<String, Json>,
+) {
+    let known_fields: &[&str] = match backend_kind {
+        "in_memory" => &["max_bytes", "max_entries"],
+        "redis" => &["url", "key_prefix"],
+        _ => return,
+    };
+    validate_unknown_fields(
+        diagnostics,
+        policy,
+        Some(format!("response_cache.backend.{backend_kind}")),
+        backend_config,
+        known_fields,
+    );
+}
+
 fn validate_backend_config_fields(
     diagnostics: &mut Vec<ConfigDiagnostic>,
     policy: &ConfigPolicy,

crates/adaptive/src/response_cache/config.rs

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Backend selection for the adaptive plugin's `response_cache` feature.
+//!
+//! The `response_cache` section struct ([`crate::config::ResponseCacheConfig`])
+//! lives in [`crate::config`] alongside the other `AdaptiveConfig` sections
+//! (`acg`, `adaptive_hints`, `tool_parallelism`). This module keeps the
+//! response-cache-specific backend config and the key-strategy constant next to
+//! the key/store code that consumes them.
+
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value as Json};
+
+/// Exact-request key strategy identifier.
+pub const KEY_STRATEGY_EXACT_REQUEST: &str = "exact_request";
+
+/// Default in-memory byte budget: 256 MiB.
+pub const DEFAULT_MAX_BYTES: usize = 256 * 1024 * 1024;
+
+/// Backend selection mirroring the adaptive [`crate::config::BackendSpec`] shape.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(default)]
+pub struct BackendConfig {
+    /// Backend kind: `"in_memory"` or `"redis"` (needs the `redis-backend` feature).
+    pub kind: String,
+    /// Backend-specific options (in_memory: `max_bytes`/`max_entries`; redis: `url`/`key_prefix`).
+    pub config: Map<String, Json>,
+}
+
+impl Default for BackendConfig {
+    fn default() -> Self {
+        Self {
+            kind: "in_memory".to_string(),
+            config: Map::new(),
+        }
+    }
+}
+
+impl BackendConfig {
+    /// In-memory total-bytes budget before oldest-first eviction.
+    pub fn max_bytes(&self) -> usize {
+        self.config
+            .get("max_bytes")
+            .and_then(Json::as_u64)
+            .map(|value| value as usize)
+            .unwrap_or(DEFAULT_MAX_BYTES)
+    }
+
+    /// Optional in-memory entry-count cap (on top of the byte budget).
+    pub fn max_entries(&self) -> Option<usize> {
+        self.config
+            .get("max_entries")
+            .and_then(Json::as_u64)
+            .map(|value| value as usize)
+    }
+}
+
+nemo_relay::editor_config! {
+    impl BackendConfig {
+        kind => { label: "kind", kind: Enum, values: ["in_memory", "redis"] },
+        config => { label: "config", kind: Json },
+    }
+}