diff --git a/.github/workflows/ci-lite.yml b/.github/workflows/ci-lite.yml index 792d00df59..a903d00bba 100644 --- a/.github/workflows/ci-lite.yml +++ b/.github/workflows/ci-lite.yml @@ -548,6 +548,8 @@ jobs: openhuman/tools/ops_tests.rs openhuman/voice/compile_status.rs openhuman/web3/stub.rs + openhuman/web3/wallet/primitives/address/evm/test.rs + openhuman/web3/wallet/primitives/address/test.rs openhuman/web3/wallet/stub.rs openhuman/web3/x402/stub.rs EOF diff --git a/.gitmodules b/.gitmodules index 084444e410..eccb4cea0f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -27,9 +27,3 @@ path = vendor/tinymemory url = https://github.com/tinyhumansai/tinymemory.git branch = main -[submodule "vendor/tinywallet"] - path = vendor/tinywallet - url = https://github.com/tinyhumansai/tinywallet -[submodule "vendor/tinydocs"] - path = vendor/tinydocs - url = https://github.com/tinyhumansai/tinydocs diff --git a/Cargo.lock b/Cargo.lock index f0019d28ef..dd25262ee8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,7 +148,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4103,6 +4103,7 @@ dependencies = [ "async-trait", "axum", "base64 0.22.1", + "bech32 0.11.1", "block2 0.6.2", "bs58", "bytes", @@ -4110,6 +4111,7 @@ dependencies = [ "chrono", "chrono-tz", "clap", + "coins-bip32", "coins-bip39", "cpal", "cron", @@ -4156,6 +4158,7 @@ dependencies = [ "regex", "reqwest", "ring", + "ripemd", "rppal", "rusqlite", "rustls", @@ -4166,6 +4169,7 @@ dependencies = [ "serde_repr", "serde_yaml", "sha2 0.10.9", + "sha3", "socketioxide", "starship-battery", "sysinfo", @@ -4177,7 +4181,6 @@ dependencies = [ "tinychannels", "tinycortex", "tinycortex-api", - "tinydocs", "tinyflows", "tinyhumans-sdk", "tinyjuice", @@ -4186,7 +4189,6 @@ dependencies = [ "tinymemory-core", "tinymemory-tinycortex", "tinyplace", - "tinywallet", "tokio", "tokio-stream", "tokio-tungstenite 0.29.0", @@ -6502,14 +6504,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "tinydocs" -version = "0.1.12" -dependencies = [ - "serde", - "thiserror 2.0.18", -] - [[package]] name = "tinyflows" version = "0.6.1" @@ -6567,7 +6561,7 @@ dependencies = [ [[package]] name = "tinymemory" -version = "0.3.0" +version = "1.0.1" dependencies = [ "anyhow", "async-trait", @@ -6698,27 +6692,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tinywallet" -version = "0.2.0" -dependencies = [ - "async-trait", - "bech32 0.11.1", - "bs58", - "coins-bip32", - "coins-bip39", - "ed25519-dalek", - "hex", - "hmac", - "ripemd", - "serde", - "serde_json", - "sha2 0.10.9", - "sha3", - "thiserror 2.0.18", - "zeroize", -] - [[package]] name = "tokio" version = "1.52.3" diff --git a/Cargo.toml b/Cargo.toml index fddd6043f7..3ed6543dd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -454,7 +454,6 @@ unicode-width = { version = "0.2", optional = true } # After cloning: `git submodule update --init vendor/tinydocs`. # # Optional: exclusive to the default-ON `documents` feature. -tinydocs = { path = "vendor/tinydocs", default-features = false, optional = true } # TinyWallet — host-agnostic multi-chain wallet primitives. Owns the address # formats themselves: parsing, validation, and the conversions between their @@ -475,11 +474,14 @@ tinydocs = { path = "vendor/tinydocs", default-features = false, optional = true # What stays is address validation, key derivation, the `Transport` seam, the # wire contract, EIP-712 hashing and ERC-20 calldata — none of which needs a # chain library. -tinywallet = { path = "vendor/tinywallet", default-features = false, features = ["btc", "evm", "solana", "tron", "keccak", "key", "net", "wire", "eip712", "abi", "x402"], optional = true } # secp256k1 signing over the digests the wallet module hands back. Pure Rust, # and already in the graph beneath `coins-bip32` (which derives the key being # used), so naming it directly costs nothing and is what lets `bitcoin` go. k256 = { version = "0.13", default-features = false, features = ["std", "ecdsa"], optional = true } +bech32 = { version = "0.11", optional = true } +ripemd = { version = "0.1", optional = true } +coins-bip32 = { version = "0.8", optional = true } +sha3 = { version = "0.10", optional = true } [target.'cfg(windows)'.dependencies] # Windows: tokio-tungstenite uses native-tls (schannel) so wss:// @@ -676,7 +678,7 @@ inference = ["dep:cpal"] # reference instead of extracted text # (`agent::multimodal::extract_pdf_text`). Slim / headless builds opt out via # `--no-default-features --features ""`. -documents = ["dep:tinydocs", "modules"] +documents = ["modules"] # The dynamic module host (`openhuman::modules`): the loader that admits a # compiled `cdylib` through tinybus's ABI descriptor, manifest, dependency and # SHA-256 gates, plus the `modules` RPC namespace and the registry of modules @@ -739,9 +741,13 @@ voice = [ # pulls them in regardless. web3 = [ "dep:k256", + "dep:bech32", + "dep:ripemd", + "dep:coins-bip32", + "dep:sha3", "dep:curve25519-dalek", "dep:coins-bip39", - "dep:tinywallet", + "modules", ] # Managed Node.js runtime: `runtime::node` (download / verify / extract / install diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 4117beda5c..a9e210f64b 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -197,7 +197,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -208,7 +208,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1746,7 +1746,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2111,7 +2111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3105,7 +3105,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.58.0", ] [[package]] @@ -4052,7 +4052,7 @@ dependencies = [ "png 0.18.1", "serde", "thiserror 2.0.20", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4235,7 +4235,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4750,12 +4750,14 @@ dependencies = [ "async-trait", "axum", "base64 0.22.1", + "bech32 0.11.1", "block2 0.6.2", "bs58", "bytes", "chacha20poly1305", "chrono", "chrono-tz", + "coins-bip32", "coins-bip39", "cpal", "cron", @@ -4794,6 +4796,7 @@ dependencies = [ "regex", "reqwest 0.12.28", "ring", + "ripemd", "rusqlite", "rustls", "schemars 1.2.2", @@ -4803,6 +4806,7 @@ dependencies = [ "serde_repr", "serde_yaml", "sha2 0.10.9", + "sha3", "socketioxide", "starship-battery", "sysinfo", @@ -4814,7 +4818,6 @@ dependencies = [ "tinychannels", "tinycortex", "tinycortex-api", - "tinydocs", "tinyflows", "tinyhumans-sdk", "tinyjuice", @@ -4823,7 +4826,6 @@ dependencies = [ "tinymemory-core", "tinymemory-tinycortex", "tinyplace", - "tinywallet", "tokio", "tokio-stream", "tokio-tungstenite 0.29.0", @@ -5462,7 +5464,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5958,7 +5960,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6016,7 +6018,7 @@ dependencies = [ "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6715,7 +6717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7480,10 +7482,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7761,14 +7763,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "tinydocs" -version = "0.1.12" -dependencies = [ - "serde", - "thiserror 2.0.20", -] - [[package]] name = "tinyflows" version = "0.6.1" @@ -7826,7 +7820,7 @@ dependencies = [ [[package]] name = "tinymemory" -version = "0.3.0" +version = "1.0.1" dependencies = [ "anyhow", "async-trait", @@ -7948,27 +7942,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tinywallet" -version = "0.2.0" -dependencies = [ - "async-trait", - "bech32 0.11.1", - "bs58", - "coins-bip32", - "coins-bip39", - "ed25519-dalek", - "hex", - "hmac", - "ripemd", - "serde", - "serde_json", - "sha2 0.10.9", - "sha3", - "thiserror 2.0.20", - "zeroize", -] - [[package]] name = "tokio" version = "1.53.1" @@ -8333,7 +8306,7 @@ dependencies = [ "png 0.18.1", "serde", "thiserror 2.0.20", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8414,7 +8387,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9077,7 +9050,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -9201,19 +9174,6 @@ dependencies = [ "windows-strings 0.4.2", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement 0.60.2", - "windows-interface 0.59.3", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - [[package]] name = "windows-future" version = "0.2.1" @@ -9351,15 +9311,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link 0.2.1", -] - [[package]] name = "windows-strings" version = "0.1.0" @@ -9379,15 +9330,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link 0.2.1", -] - [[package]] name = "windows-sys" version = "0.45.0" diff --git a/docs/specs/memory-guard-allowlist.md b/docs/specs/memory-guard-allowlist.md index a6fee3fdcf..f9590daf13 100644 --- a/docs/specs/memory-guard-allowlist.md +++ b/docs/specs/memory-guard-allowlist.md @@ -17,7 +17,7 @@ dead-string rot the ratchet exists to prevent. ## Scope -The lint scans `src/` for thirteen patterns, keyed on `(file, pattern)` so the +The lint scans `src/` for twelve patterns, keyed on `(file, pattern)` so the failure message names the needle that tripped: | Pattern | What it hands out | @@ -28,7 +28,7 @@ failure message names the needle that tripped: | `.profile_conn(` | raw `Arc>` (one in-family site) | | `.profile_store(` | a typed `ProfileStore` — confined, but still unguarded | | `.get_document(` | `pub(crate)` read-one escape hatch | -| `EmbeddedMemoryProvider::new(` / `NullMemoryProvider::new(` | a driver, built outside `binding::for_workspace` | +| `NullMemoryProvider::new(` | a driver, built outside `binding::for_workspace` | | `MemoryClient::from_workspace_dir(` | a second engine on the same store | | `binding::for_workspace(` / `.memory_binding(` | a raw `MemoryBinding` | | `.unguarded_provider(` | the raw `Arc` off a `MemoryBinding` | @@ -98,8 +98,6 @@ changes anything here. | Path | Reason | | --- | --- | -| `memory/driver/embedded/mod.rs` | This **is** the driver. Guarding it would be a cycle. | -| `memory/driver/embedded/tool_memory_tests.rs` | Driver tests. | | `memory/tinycortex/sync.rs` | The engine seam. | | `memory/global.rs` | The process-global slot itself. | | `memory/ops/helpers.rs` | Defines `active_memory_client`. | @@ -145,18 +143,14 @@ are recorded here so M4c starts from the real set. | `agent/harness/session/builder/factory.rs` | `.memory_handle()` → `Arc`. | | `flows/tinyflows/memory_adapter.rs` | Returns `Arc` to satisfy a tinyflows engine trait. The contract has no `Arc` door. | | `flows/bus.rs` | `resolve_memory() -> Option>`, and carries a `#[cfg(test)] memory_override` seam a guard would bypass. | -| `memory/ops/tool_memory.rs` (`open_store`) | Still needed by the four handlers left on the client. Shrank; did not disappear. | ### D. No contract method exists, or the wire shape would change | Path | Reason | | --- | --- | -| `memory/ops/documents.rs` — `namespace_list`, `doc_ingest`, `doc_list`, `doc_delete`, `clear_namespace`, `context_query`, `context_recall`, `memory_*` | Each answers with a `serde_json::Value` / `String` shape with no typed contract twin; `clear_namespace` has no contract method at all; `memory_query_namespace` depends on `query_limit_for_request(client: &MemoryClient, …)`. | -| `memory/ops/kv_graph.rs` — `kv_get`, `kv_delete`, `kv_list_namespace`, `graph_upsert`, `graph_query` | `kv_get` is an O(slice) scan in the driver and returns `MemoryKvRecord`, not `Value`; `kv_delete` has **no** contract method; `graph_query`'s camelCase→typed conversion is documented as new and lossy. | -| `memory/ops/tool_memory.rs` — `tool_rule_put`, `tool_rule_get`, `tool_rules_json`, `tool_rules_for_prompt` | `put_tool_rule` returns unit while the RPC returns the stored rule with a refreshed `updated_at`; the other three have no contract equivalent. | +| `memory/ops/documents.rs` — `doc_ingest` and retrieval envelope handlers | These still depend on engine-only ingestion and retrieval shapes. Namespace/document listing, deletion, context query, and context recall now use the shared Documents API. | | `memory/ops/sync.rs` | `client.ingestion_state().snapshot()` — queue telemetry, absent from the contract. | -| `memory/ops/learn.rs` | `list_namespaces() -> Vec` vs the contract's `Vec`, then heavy engine work. | -| `flows/ops.rs` | `clear_namespace` (no contract method) plus a `memory_client_override` test seam. | +| `flows/ops.rs` | The production namespace clear uses `MemoryDocuments`; only the directly injected `MemoryClientRef` test seam remains raw. | | `integrations/composio/schemas.rs` | Passes `&MemoryClientRef` into `user_scopes::save`. | | `memory/sync/composio/providers/user_scopes.rs`, `types.rs` | Same `&MemoryClientRef` parameter shape. | @@ -168,16 +162,15 @@ module). ## Honest scorecard -Six of the twenty-eight `active_memory_client()` call sites now route through -the guard — four RPC handlers plus the `memory_tools_list` / `memory_tools_put` -agent tools. Raw `profile_conn()` no longer leaves the memory family — but the ten -profile/facet call sites it fed are still unguarded, now through a typed -`ProfileStore`, and twelve non-test `memory_handle()` sites still hand out raw -handles. The defensible claim is therefore: +The document listing/mutation handlers, the full KV/graph handler family, the +tool-memory handlers, and flow namespace cleanup now use the shared memory API. +Raw profile/facet access and consumers whose foreign traits require +`Arc` remain unguarded and are enumerated above. The defensible +claim is therefore: -> Every memory RPC handler whose contract twin is a literal delegation now -> routes through the guard, and every remaining bypass is enumerated here with -> a reason and pinned by a drift guard. +> Every memory RPC handler covered by a shared capability family routes through +> the guard, and every remaining bypass is enumerated here with a reason and +> pinned by a drift guard. "Impossible to skip by construction" is **not** true until `memory_handle()` is gone and the profile/facet tables have a capability family to be guarded diff --git a/src/core/all.rs b/src/core/all.rs index afd370e653..f8165a388e 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -10,7 +10,7 @@ use std::sync::OnceLock; use serde_json::{Map, Value}; -use tinycortex_api::capabilities::{Capabilities, Capability}; +use crate::openhuman::memory::api::capabilities::{Capabilities, Capability}; use crate::core::ControllerSchema; @@ -242,7 +242,7 @@ struct GroupedController { /// Absence, not a stub that errors — a registered-but-failing method /// teaches a model that the capability exists and makes it retry. Same /// reasoning as the `flows` compile-time gate (see CLAUDE.md) and as - /// `tinycortex_api::capabilities`' module docs. + /// `crate::openhuman::memory::api::capabilities`' module docs. capability: Option, controller: RegisteredController, } diff --git a/src/core/all_tests.rs b/src/core/all_tests.rs index c99c85c0de..955c298d93 100644 --- a/src/core/all_tests.rs +++ b/src/core/all_tests.rs @@ -1658,7 +1658,7 @@ fn memory_controllers_form_one_contiguous_run_in_aggregator_order() { // present and failing, because a registered-but-failing method teaches a model // the capability exists and makes it retry. -use tinycortex_api::capabilities::Capability; +use crate::openhuman::memory::api::capabilities::Capability; /// A workspace path unique to one test. /// @@ -1983,8 +1983,9 @@ async fn visible_under( } #[tokio::test] +#[cfg(feature = "modules")] async fn memory_families_registered_when_capabilities_advertised() { - // The embedded `tinycortex` driver (the default config) advertises + // The TinyMemory module driver advertises // `Capabilities::all()`, so every gated family is present. Scoped rather // than unscoped so this proves a BOUND driver's set, not the unbound // default-open fallback. diff --git a/src/core/cli_capability.rs b/src/core/cli_capability.rs index 486c20bc76..2ddb1043a1 100644 --- a/src/core/cli_capability.rs +++ b/src/core/cli_capability.rs @@ -37,8 +37,8 @@ //! static namespace/function names, never from user-supplied argument values, //! and no memory content can reach this path at all. +use crate::openhuman::memory::api::capabilities::{Capabilities, Capability}; use anyhow::Result; -use tinycortex_api::capabilities::{Capabilities, Capability}; use crate::core::subsystem::DriverClass; diff --git a/src/core/cli_capability_tests.rs b/src/core/cli_capability_tests.rs index da1e7f547a..145b9a099f 100644 --- a/src/core/cli_capability_tests.rs +++ b/src/core/cli_capability_tests.rs @@ -92,10 +92,13 @@ fn message_never_contains_a_credential_or_endpoint() { } #[test] -fn bound_driver_probe_reports_the_default_embedded_driver() { +fn bound_driver_probe_reports_the_default_module_driver() { let cfg = MemorySubsystemConfig::default(); let binding = binding_for("default", cfg.clone()); - assert_eq!(binding.driver_id(), cfg.driver); + assert_eq!( + binding.driver_id(), + crate::openhuman::memory::binding::MODULE_ID + ); assert_eq!(binding.capabilities(), Capabilities::all()); } diff --git a/src/core/cli_tests.rs b/src/core/cli_tests.rs index 5c56756278..e03bf0f6d5 100644 --- a/src/core/cli_tests.rs +++ b/src/core/cli_tests.rs @@ -330,7 +330,7 @@ use crate::core::all::{ capability_for_parts, capability_for_rpc_method, sole_capability_for_namespace, }; use crate::core::cli_capability::capability_verdict; -use tinycortex_api::capabilities::Capabilities; +use crate::openhuman::memory::api::capabilities::Capabilities; #[test] fn capability_gated_namespace_reports_a_config_fact_not_a_typo() { @@ -368,7 +368,9 @@ fn capability_gated_function_reports_a_config_fact_not_a_typo() { fn capability_gated_rpc_method_reports_its_family_unfiltered() { assert_eq!( capability_for_rpc_method("openhuman.memory_tree_wipe_all"), - Some(Some(tinycortex_api::capabilities::Capability::Tree)) + Some(Some( + crate::openhuman::memory::api::capabilities::Capability::Tree + )) ); } diff --git a/src/core/memory_cli.rs b/src/core/memory_cli.rs index df83c45c97..11ea516ab4 100644 --- a/src/core/memory_cli.rs +++ b/src/core/memory_cli.rs @@ -62,7 +62,9 @@ const SUBCOMMAND_CONTROLLER: &[(&str, &str)] = &[ /// The capability `openhuman memory ` needs, if any. Resolved from the /// controller registry, never from a local table. -fn required_capability(subcommand: &str) -> Option { +fn required_capability( + subcommand: &str, +) -> Option { let function = SUBCOMMAND_CONTROLLER .iter() .find(|(sub, _)| *sub == subcommand) @@ -546,7 +548,7 @@ mod tests { use super::*; use crate::core::cli_capability::{capability_verdict, CAPABILITY_UNAVAILABLE_PREFIX}; use crate::core::subsystem::DriverClass; - use tinycortex_api::capabilities::{Capabilities, Capability}; + use crate::openhuman::memory::api::capabilities::{Capabilities, Capability}; /// Drift guard: a renamed controller function must break here rather than /// silently un-gate a subcommand (`required_capability` would start diff --git a/src/core/runtime/context.rs b/src/core/runtime/context.rs index de27faec16..8e52d224c3 100644 --- a/src/core/runtime/context.rs +++ b/src/core/runtime/context.rs @@ -278,11 +278,11 @@ impl CoreContext { /// would keep `memory_store` / `memory_recall` / `memory.list_documents` /// answering off the embedded store the guarded re-point has not yet /// covered. See [`MemoryBinding::disables_memory`](crate::openhuman::memory::binding::MemoryBinding::disables_memory). - pub fn memory_capabilities(&self) -> tinycortex_api::capabilities::Capabilities { + pub fn memory_capabilities(&self) -> crate::openhuman::memory::api::capabilities::Capabilities { self.memory_binding() .map(|binding| { if binding.disables_memory() { - tinycortex_api::capabilities::Capabilities::default() + crate::openhuman::memory::api::capabilities::Capabilities::default() } else { binding.capabilities() } @@ -317,7 +317,8 @@ impl CoreContext { /// there is no context at all. This is the direct analogue of /// `core::all::group_allowed` and is the function a future capability /// registration filter calls. - pub fn current_memory_capabilities() -> tinycortex_api::capabilities::Capabilities { + pub fn current_memory_capabilities() -> crate::openhuman::memory::api::capabilities::Capabilities + { Self::current() .map(|ctx| ctx.memory_capabilities()) .unwrap_or_else(crate::openhuman::memory::binding::unbound_default_capabilities) @@ -972,10 +973,12 @@ mod tests { }; let bind_a = ctx.memory_binding().expect("bind workspace A"); - assert_eq!( - bind_a.class(), - crate::core::subsystem::DriverClass::Embedded - ); + let expected = if cfg!(feature = "modules") { + crate::core::subsystem::DriverClass::Module + } else { + crate::core::subsystem::DriverClass::Null + }; + assert_eq!(bind_a.class(), expected); let null_cfg = crate::openhuman::config::schema::MemorySubsystemConfig { driver: "null".to_string(), @@ -1016,7 +1019,7 @@ mod tests { }; let bind_a = a.memory_binding().expect("bind workspace A"); - assert_eq!(bind_a.driver_id(), "tinycortex"); + assert_eq!(bind_a.driver_id(), "tinymemory"); assert!(bind_a.fallback().is_none()); let bind_b = b.memory_binding().expect("workspace B falls back"); @@ -1045,7 +1048,7 @@ mod tests { assert!(ctx.memory_binding().is_err(), "no workspace ⇒ no binding"); assert_eq!( ctx.memory_capabilities(), - tinycortex_api::capabilities::Capabilities::all(), + crate::openhuman::memory::api::capabilities::Capabilities::all(), "a context with no binding must not deny any capability" ); } @@ -1060,14 +1063,14 @@ mod tests { fn current_memory_capabilities_defaults_open_without_a_context() { assert_eq!( crate::openhuman::memory::binding::unbound_default_capabilities(), - tinycortex_api::capabilities::Capabilities::all() + crate::openhuman::memory::api::capabilities::Capabilities::all() ); // And when a context *is* ambient, the call resolves through it rather // than erroring. let ctx = CoreContext::for_test(crate::core::runtime::DomainSet::full(), None, None); assert_eq!( ctx.memory_capabilities(), - tinycortex_api::capabilities::Capabilities::all() + crate::openhuman::memory::api::capabilities::Capabilities::all() ); } @@ -1078,7 +1081,7 @@ mod tests { let ctx = CoreContext::for_test(crate::core::runtime::DomainSet::harness(), None, None); assert_eq!( ctx.memory_capabilities(), - tinycortex_api::capabilities::Capabilities::all() + crate::openhuman::memory::api::capabilities::Capabilities::all() ); } } diff --git a/src/core/subsystem/driver.rs b/src/core/subsystem/driver.rs index ae23a8ac34..1a75489b20 100644 --- a/src/core/subsystem/driver.rs +++ b/src/core/subsystem/driver.rs @@ -34,7 +34,7 @@ use serde::{Deserialize, Serialize}; /// This is a **host configuration fact**, never something the driver reports. /// /// Deliberately not `#[non_exhaustive]`, for the same reason -/// `tinycortex_api::capabilities::Capability` is not: adding a class must break +/// `crate::openhuman::memory::api::capabilities::Capability` is not: adding a class must break /// every exhaustive `match` in the host, because those matches are where policy /// (egress, trust, credential resolution) is decided per class. #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] @@ -46,27 +46,7 @@ pub enum DriverClass { /// An out-of-process backend reached through a transport adapter over a /// documented wire contract. External, - /// A loadable native module: a `cdylib` admitted through tinybus's ABI, - /// manifest and digest gates and reached over the in-process module bus. - /// - /// Neither of the two classes above fits, and the difference is the policy - /// this class gates: - /// - /// - not [`Self::Embedded`], because the code is **not compiled into this - /// binary**. It is downloaded, verified against a digest pinned in - /// `modules::registry`, and `dlopen`ed. Whether it is present at all is a - /// runtime fact, so a capability set derived from it can be empty on a - /// platform no artifact is published for. - /// - not [`Self::External`], because there is **no egress and no process - /// boundary**. It shares this address space, these privileges and this - /// crash domain, so the trust checks that make sense for a remote backend - /// (endpoint allowlisting, TLS, credential scoping) are neither applicable - /// nor sufficient. What protects the host is admission, not isolation. - /// - /// Treating a module as `External` would apply egress policy to something - /// that makes no network calls while implying an isolation the loader does - /// not provide; treating it as `Embedded` would claim a compile-time - /// guarantee that a downloaded artifact does not have. + /// A verified native TinyBus module loaded into this process. Module, /// A stub advertising zero capabilities — what a compiled-out or /// unconfigured subsystem binds to. @@ -128,7 +108,7 @@ impl std::str::FromStr for DriverClass { /// Liveness of a bound driver, in the kernel's generic vocabulary. /// -/// Shaped one-for-one against `tinycortex_api::health::MemoryHealth` — and +/// Shaped one-for-one against `crate::openhuman::memory::api::health::MemoryHealth` — and /// against whatever the next subsystem's contract carries — so the boundary /// conversion is a total three-arm `match` that cannot drift. Serializes as an /// internally-tagged object with a stable snake_case `status` discriminant: @@ -211,7 +191,7 @@ impl std::fmt::Display for DriverHealth { /// The kernel deliberately does not know any subsystem's family vocabulary — /// `"tree"` and `"tool_memory"` mean something to the memory subsystem and /// nothing here. Each subsystem's adapter converts its own typed set (for -/// memory: `tinycortex_api::capabilities::Capabilities`) into this at bind +/// memory: `crate::openhuman::memory::api::capabilities::Capabilities`) into this at bind /// time, and the kernel only ever asks "does the bound driver advertise this /// string?" when deciding whether to register a controller or emit a tool. /// diff --git a/src/core/subsystem/driver_tests.rs b/src/core/subsystem/driver_tests.rs index 0c140f6b5f..1b8a557689 100644 --- a/src/core/subsystem/driver_tests.rs +++ b/src/core/subsystem/driver_tests.rs @@ -97,7 +97,7 @@ fn driver_health_display_includes_the_reason() { /// silently making the conversion partial. #[test] fn driver_health_shape_matches_memory_health_one_for_one() { - use tinycortex_api::health::MemoryHealth; + use crate::openhuman::memory::api::health::MemoryHealth; let pairs: Vec<(MemoryHealth, DriverHealth)> = vec![ (MemoryHealth::Ready, DriverHealth::Ready), @@ -188,7 +188,7 @@ fn driver_capabilities_contains_all_is_subset_semantics() { /// memory capability is. #[test] fn every_memory_contract_capability_string_maps_into_driver_capabilities() { - use tinycortex_api::capabilities::Capability; + use crate::openhuman::memory::api::capabilities::Capability; let caps: DriverCapabilities = Capability::ALL.iter().map(|cap| cap.as_str()).collect(); diff --git a/src/core/subsystem/mod.rs b/src/core/subsystem/mod.rs index eaac2a91ad..53fb1de1bb 100644 --- a/src/core/subsystem/mod.rs +++ b/src/core/subsystem/mod.rs @@ -13,7 +13,7 @@ //! //! Since M2b the memory adapter exists in //! [`crate::openhuman::memory::binding`] (it converts -//! `tinycortex_api::MemoryHealth` into [`DriverHealth`] and the contract's +//! `crate::openhuman::memory::api::MemoryHealth` into [`DriverHealth`] and the contract's //! typed capability set into [`DriverCapabilities`]), and M2c added the //! read-only [`status`] projection plus the `subsystems` RPC namespace and the //! `openhuman subsystems` CLI table. diff --git a/src/core/subsystem/status.rs b/src/core/subsystem/status.rs index 9b419ae119..00d6cacf02 100644 --- a/src/core/subsystem/status.rs +++ b/src/core/subsystem/status.rs @@ -8,7 +8,7 @@ //! //! ## Capabilities cross the wire as opaque strings, never as a typed set //! -//! A memory driver's typed set (`tinycortex_api::capabilities::Capabilities`) +//! A memory driver's typed set (`crate::openhuman::memory::api::capabilities::Capabilities`) //! is a `u16` bitset whose `Deserialize` rejects the **whole** array on a //! single unrecognised family string. A driver speaking a newer minor contract //! may legitimately advertise a family this build has never heard of, and diff --git a/src/openhuman/agent/artifacts/ops_tests.rs b/src/openhuman/agent/artifacts/ops_tests.rs index 7539598c90..8b042a98c0 100644 --- a/src/openhuman/agent/artifacts/ops_tests.rs +++ b/src/openhuman/agent/artifacts/ops_tests.rs @@ -248,6 +248,7 @@ async fn regenerate_errors_when_args_missing() { #[cfg(feature = "documents")] #[tokio::test] +#[ignore = "needs a built tinydocs module and its own process; presentation module E2E covers generation"] async fn regenerate_reruns_producer_and_reuses_id() { use crate::openhuman::agent::artifacts::store::{ create_artifact, get_artifact, save_artifact_args, diff --git a/src/openhuman/config/schema/load_tests.rs b/src/openhuman/config/schema/load_tests.rs index 0b4ea74832..e2f5c11a7c 100644 --- a/src/openhuman/config/schema/load_tests.rs +++ b/src/openhuman/config/schema/load_tests.rs @@ -813,6 +813,8 @@ fn env_overlay_memory_sync_interval_parses_and_honours_zero() { #[test] fn env_overlay_subsystems_memory_driver_and_hooks_apply() { let mut cfg = Config::default(); + // The shared schema retains the persisted legacy id; binding normalizes it + // to the built-in `tinymemory` module id. assert_eq!(cfg.subsystems.memory.driver, "tinycortex"); assert!(cfg.subsystems.memory.hooks.auto_recall); assert!(cfg.subsystems.memory.hooks.auto_capture); diff --git a/src/openhuman/flows/agents/workflow_builder/prompt.md b/src/openhuman/flows/agents/workflow_builder/prompt.md index 1caefdabce..de960f6cd4 100644 --- a/src/openhuman/flows/agents/workflow_builder/prompt.md +++ b/src/openhuman/flows/agents/workflow_builder/prompt.md @@ -305,7 +305,7 @@ A `WorkflowGraph` is `{ name?, nodes: [...], edges: [...] }`. - **Exactly ONE `trigger` node is required.** Every other node should be reachable from it; a dry-run helps catch orphans. -### The 15 node kinds +### The 16 node kinds > The authoritative, always-current config shapes, ports, examples, and gotchas > for each kind live in the `list_node_kinds` / `get_node_kind_contract { kind }` @@ -531,8 +531,9 @@ A `WorkflowGraph` is `{ name?, nodes: [...], edges: [...] }`. media generation, files, …). No `connection_ref`. Args go in `config.args`. 4. **`http_request`** — `config.method` + `config.url`, optional `headers` / `body`; `config.connection_ref` = an `http_cred:` for auth. -5. **`code`** — `config.language` (`"javascript"` | `"python"`) + `config.source`. -6. **`condition`** — boolean gate on `config.field`; routes to the **`true`** or +5. **`shell`** — a shell command in `config.source`. +6. **`code`** — `config.language` (`"javascript"` | `"python"`) + `config.source`. +7. **`condition`** — boolean gate on `config.field`; routes to the **`true`** or **`false`** port. Wire both (or the `false` branch dead-ends). If `config.field` binds to an `agent` node's output, that field's `output_parser.schema` property MUST be declared `"type": "boolean"` (see @@ -554,21 +555,21 @@ A `WorkflowGraph` is `{ name?, nodes: [...], edges: [...] }`. enforced: `propose_workflow`/`revise_workflow`/`save_workflow` reject a `condition` node whose outgoing edges don't emit `"true"`/`"false"` on `from_port`. -7. **`switch`** — multi-way on `config.expression` or `config.field`; routes to +8. **`switch`** — multi-way on `config.expression` or `config.field`; routes to the matching **case** port, else **`default`**. -8. **`merge`** — fan-in barrier; passes inputs through. No config. -9. **`split_out`** — `config.path` to an array field; fans out one item per +9. **`merge`** — fan-in barrier; passes inputs through. No config. +10. **`split_out`** — `config.path` to an array field; fans out one item per element. -10. **`transform`** — `config.set` = `{ key: "=expr" }`, merged onto each item. -11. **`output_parser`** — passthrough today; no config required. -12. **`sub_workflow`** — `config.workflow` = an embedded child `WorkflowGraph`. -13. **`memory`** — reads or writes host-managed memory directly, no agent turn +11. **`transform`** — `config.set` = `{ key: "=expr" }`, merged onto each item. +12. **`output_parser`** — passthrough today; no config required. +13. **`sub_workflow`** — `config.workflow` = an embedded child `WorkflowGraph`. +14. **`memory`** — reads or writes host-managed memory directly, no agent turn involved. See "The `memory` node" just below for the full reference. -14. **`dedup`** — commit-on-success exactly-once filter: drops an item whose +15. **`dedup`** — commit-on-success exactly-once filter: drops an item whose per-item key was already committed by a prior successful run. See "The `dedup` node" below — this is THE way to do "process each item once", not a memory recall/condition graph. -15. **`loop`** — a bounded loop head. Emits on `body` while it keeps looping +16. **`loop`** — a bounded loop head. Emits on `body` while it keeps looping and on `done` when it stops; you CLOSE THE LOOP yourself by wiring the body's last node back to the loop node. `config.max_iterations` is optional and always finite, defaulting to 25; `config.on_exceeded` is `"error"` diff --git a/src/openhuman/flows/ops.rs b/src/openhuman/flows/ops.rs index 27b913acd7..189d530bf9 100644 --- a/src/openhuman/flows/ops.rs +++ b/src/openhuman/flows/ops.rs @@ -23,6 +23,7 @@ use crate::openhuman::flows::types::{ FlowConnection, FlowRunStep, FlowRunTrigger, FlowSuggestion, SuggestionStatus, }; use crate::openhuman::flows::{flow_namespace, Flow, FlowRun}; +use crate::openhuman::memory::api::provider::MemoryProvider; use crate::openhuman::memory::store::MemoryClientRef; use crate::openhuman::security::approval::{ ApprovalChatContext, FlowRunContext, APPROVAL_CHAT_CONTEXT, APPROVAL_COPILOT_STREAM_CONTEXT, @@ -4199,19 +4200,25 @@ async fn flows_delete_impl( // entries or run digests behind. Never fails the delete itself: the flow // row is already gone by this point regardless of what happens here. let memory_namespace = flow_namespace(id); - let client_result = match memory_client_override { - Some(client) => Ok(client), - None => crate::openhuman::memory::ops::helpers::active_memory_client().await, - }; - match client_result { - Ok(client) => { - if let Err(e) = client.clear_namespace(&memory_namespace).await { - tracing::warn!(target: "flows", flow_id = %id, namespace = %memory_namespace, error = %e, "[flows] flows_delete: failed to clear flow memory namespace"); - } - } - Err(e) => { - tracing::warn!(target: "flows", flow_id = %id, namespace = %memory_namespace, error = %e, "[flows] flows_delete: memory client unavailable — could not clear flow memory namespace"); + let clear_result = if let Some(client) = memory_client_override { + client + .clear_namespace(&memory_namespace) + .await + .map_err(|error| error.to_string()) + } else { + match crate::openhuman::memory::ops::guard::active_memory_guard().await { + Ok(guard) => match guard.as_documents() { + Some(documents) => documents + .clear_namespace(&memory_namespace) + .await + .map_err(|error| error.to_string()), + None => Err("memory driver does not support the documents family".to_string()), + }, + Err(error) => Err(error), } + }; + if let Err(error) = clear_result { + tracing::warn!(target: "flows", flow_id = %id, namespace = %memory_namespace, %error, "[flows] flows_delete: failed to clear flow memory namespace"); } publish_flow_changed(id, "deleted", "system"); diff --git a/src/openhuman/flows/ops_tests.rs b/src/openhuman/flows/ops_tests.rs index affba8812a..4923badd92 100644 --- a/src/openhuman/flows/ops_tests.rs +++ b/src/openhuman/flows/ops_tests.rs @@ -1588,6 +1588,7 @@ async fn flows_delete_clears_flow_memory_namespace() { let tmp = TempDir::new().unwrap(); let config = test_config(&tmp); + crate::openhuman::memory::host_impls::install_for_tests(); // A directly-constructed `MemoryClient`, injected via `flows_delete_impl` // below, instead of `memory::global` — that singleton is a single diff --git a/src/openhuman/memory/api/capabilities.rs b/src/openhuman/memory/api/capabilities.rs new file mode 100644 index 0000000000..4dd3aef7c9 --- /dev/null +++ b/src/openhuman/memory/api/capabilities.rs @@ -0,0 +1,397 @@ +//! Capability families a memory driver may advertise, and the set type used to +//! negotiate them. +//! +//! ## Why capabilities exist +//! +//! A memory driver is not required to implement the whole surface. The kernel +//! asks a driver which families it supports **once**, at bind time, caches the +//! answer, and then unregisters the RPC methods and omits the agent tools that +//! belong to an unadvertised family. Absence beats a registered handler that +//! returns "not implemented": a present-but-failing method teaches a model that +//! the capability exists and makes it retry. +//! +//! Calling an unadvertised capability is therefore a *kernel* bug, not a driver +//! error. [`crate::openhuman::memory::api::error::MemoryError::Unsupported`] exists for the one case the +//! kernel cannot pre-empt: an out-of-process driver that answers `501` for a +//! family its handshake claimed. +//! +//! ## Mandatory families +//! +//! [`Capability::Core`], [`Capability::Recall`], and [`Capability::Portability`] +//! are mandatory. Without core and recall a driver is not a memory backend at +//! all; without portability a user cannot leave it, which makes the binding a +//! one-way door. [`Capabilities::validate`] is the single place that rule is +//! encoded — call it at bind time and refuse the bind on `Err`. +//! +//! ## Wire stability +//! +//! The set crosses the process boundary in the driver handshake +//! (`POST /v1/handshake` → `{ contract_version, driver_id, capabilities[] }`), +//! so the serialized form is a JSON **array of stable snake_case strings**, not +//! discriminant integers — inserting a variant in the middle of the enum must +//! not silently re-map an already-deployed driver's advertised set. +//! [`Capability::as_str`] is the authority for those strings and is pinned +//! against the serde derive by a test. +//! +//! ## Deliberately not `#[non_exhaustive]` +//! +//! Adding a family is a [`crate::openhuman::memory::api::CONTRACT_VERSION`] **minor** bump and should +//! break every exhaustive `match` in every host that filters registration by +//! family — that compile error is the mechanism which guarantees the new family +//! is actually wired somewhere. Marking this enum `#[non_exhaustive]` would +//! convert that compile-time guarantee into a silent fall-through at the crate +//! boundary (the failure mode recorded for `DataSource` during the M0 +//! carve-out). If a future family must be added without breaking downstream +//! matches, bump the **major** version instead. + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::openhuman::memory::api::error::MemoryError; + +/// One capability family a memory driver may advertise. +/// +/// The variants are exactly the thirteen families of the memory contract. Each +/// maps to a trait family in the contract, a group of RPC methods, and a group +/// of agent tools; a driver that does not advertise a family simply has that +/// surface absent. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Capability { + /// Store / get / forget / list / namespaces. **Mandatory.** + Core, + /// Ranked retrieval for a query. **Mandatory.** + Recall, + /// Document and chat ingestion — the driver owns chunking and embedding. + Ingest, + /// The namespace-document tier: put / get / query documents. + Documents, + /// Summary-tree query, drill-down, seal, and cascade. + Tree, + /// Entity index, entity edges, and hotness. + Entities, + /// Key/value graph read and write. + Graph, + /// Snapshot capture and change computation. + Diff, + /// Goal extraction and goal records. + Goals, + /// Per-tool learned memory. + ToolMemory, + /// Accepting synced source items; the host still owns credentials and + /// scheduling. + Sources, + /// Re-embed, compact, consolidate ("dream"), and doctor. + Maintenance, + /// Export and import of the whole store as a stream. **Mandatory.** + Portability, +} + +impl Capability { + /// Every family, in declaration order. + /// + /// Declaration order is also bit order in [`Capabilities`] and iteration + /// order in its serialized form, so this slice is the single ordering + /// authority for the whole module. + pub const ALL: [Capability; 13] = [ + Capability::Core, + Capability::Recall, + Capability::Ingest, + Capability::Documents, + Capability::Tree, + Capability::Entities, + Capability::Graph, + Capability::Diff, + Capability::Goals, + Capability::ToolMemory, + Capability::Sources, + Capability::Maintenance, + Capability::Portability, + ]; + + /// The families a driver must advertise to be bindable at all. + /// + /// See the module docs for why these three and not others. + pub const MANDATORY: [Capability; 3] = [ + Capability::Core, + Capability::Recall, + Capability::Portability, + ]; + + /// Every family, in declaration order. Slice form of [`Self::ALL`], for + /// callers that want to iterate without naming the array length. + pub fn all() -> &'static [Capability] { + &Self::ALL + } + + /// Stable snake_case identifier used on the wire, in config, and in logs. + /// + /// This is the authority for the serialized form; the serde derive is + /// pinned against it by `capability_as_str_matches_serde_representation`. + /// Changing a string here is a breaking change for every already-deployed + /// driver and requires a [`crate::openhuman::memory::api::CONTRACT_VERSION`] major bump. + pub fn as_str(self) -> &'static str { + match self { + Self::Core => "core", + Self::Recall => "recall", + Self::Ingest => "ingest", + Self::Documents => "documents", + Self::Tree => "tree", + Self::Entities => "entities", + Self::Graph => "graph", + Self::Diff => "diff", + Self::Goals => "goals", + Self::ToolMemory => "tool_memory", + Self::Sources => "sources", + Self::Maintenance => "maintenance", + Self::Portability => "portability", + } + } + + /// Parse back from the on-wire form. + /// + /// # Errors + /// + /// Returns the unrecognised input in an error message. An unknown string is + /// expected in practice: a driver speaking a newer minor contract version + /// may advertise a family this build has never heard of. Callers + /// negotiating a handshake should **skip** unknown families rather than + /// fail the bind — an unknown family is one this kernel would never call. + pub fn parse(raw: &str) -> Result { + Self::ALL + .iter() + .copied() + .find(|cap| cap.as_str() == raw) + .ok_or_else(|| format!("unknown memory capability: {raw}")) + } + + /// Whether this family is mandatory for every driver. + pub fn is_mandatory(self) -> bool { + Self::MANDATORY.contains(&self) + } + + /// Position of this family in [`Self::ALL`]; also its bit index in + /// [`Capabilities`]. + fn index(self) -> u16 { + match self { + Self::Core => 0, + Self::Recall => 1, + Self::Ingest => 2, + Self::Documents => 3, + Self::Tree => 4, + Self::Entities => 5, + Self::Graph => 6, + Self::Diff => 7, + Self::Goals => 8, + Self::ToolMemory => 9, + Self::Sources => 10, + Self::Maintenance => 11, + Self::Portability => 12, + } + } + + /// Single-bit mask for this family within a [`Capabilities`] set. + fn bit(self) -> u64 { + 1u64 << self.index() + } +} + +impl std::fmt::Display for Capability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for Capability { + type Err = String; + + fn from_str(raw: &str) -> Result { + Self::parse(raw) + } +} + +/// A driver's advertised capability set. +/// +/// Internally a bitset, so `contains` is a single mask test on the hot path and +/// the type is `Copy`. Externally it serializes as a JSON array of +/// [`Capability::as_str`] strings in [`Capability::ALL`] order — duplicates in +/// the input collapse, and ordering in the input is not preserved, because a +/// set has neither. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Capabilities { + bits: u64, +} + +impl Capabilities { + /// The empty default capability set. The `null` driver advertises + /// [`Self::mandatory`] via its [`MemoryProvider::capabilities`](crate::openhuman::memory::api::provider::MemoryProvider::capabilities) + /// implementation, not this. + pub const fn empty() -> Self { + Self { bits: 0 } + } + + /// Every family. Advertised by the embedded `tinycortex` driver. + pub fn all() -> Self { + Capability::ALL.into_iter().collect() + } + + /// Exactly the mandatory families — the minimum bindable set. + pub fn mandatory() -> Self { + Capability::MANDATORY.into_iter().collect() + } + + /// Whether `capability` is advertised. + pub fn contains(&self, capability: Capability) -> bool { + self.bits & capability.bit() != 0 + } + + /// Whether every family in `other` is advertised here. + pub fn contains_all(&self, other: Capabilities) -> bool { + self.bits & other.bits == other.bits + } + + /// Adds `capability` in place. Idempotent. + pub fn insert(&mut self, capability: Capability) { + self.bits |= capability.bit(); + } + + /// Removes `capability` in place. Idempotent. + pub fn remove(&mut self, capability: Capability) { + self.bits &= !capability.bit(); + } + + /// Builder form of [`Self::insert`]. + pub fn with(mut self, capability: Capability) -> Self { + self.insert(capability); + self + } + + /// Builder form of [`Self::remove`]. + pub fn without(mut self, capability: Capability) -> Self { + self.remove(capability); + self + } + + /// Advertised families in [`Capability::ALL`] order. + pub fn iter(&self) -> impl Iterator + '_ { + Capability::ALL + .into_iter() + .filter(move |cap| self.contains(*cap)) + } + + /// Number of advertised families. + pub fn len(&self) -> usize { + self.bits.count_ones() as usize + } + + /// Whether no family is advertised. + pub fn is_empty(&self) -> bool { + self.bits == 0 + } + + /// Mandatory families this set is missing, in [`Capability::ALL`] order. + /// Empty when the set is bindable. + pub fn missing_mandatory(&self) -> Vec { + Capability::MANDATORY + .into_iter() + .filter(|cap| !self.contains(*cap)) + .collect() + } + + /// Rejects a set that is missing any mandatory family. + /// + /// Call this at bind time; on `Err` refuse the bind and fall back to the + /// embedded default rather than binding a driver a user could not leave. + /// + /// # Errors + /// + /// Returns [`MissingMandatoryCapabilities`] listing **every** missing + /// mandatory family, not just the first, so the operator sees the whole gap + /// in one message. + pub fn validate(&self) -> Result<(), MissingMandatoryCapabilities> { + let missing = self.missing_mandatory(); + if missing.is_empty() { + Ok(()) + } else { + Err(MissingMandatoryCapabilities { missing }) + } + } +} + +impl FromIterator for Capabilities { + fn from_iter>(iter: I) -> Self { + let mut set = Self::empty(); + for capability in iter { + set.insert(capability); + } + set + } +} + +impl Extend for Capabilities { + fn extend>(&mut self, iter: I) { + for capability in iter { + self.insert(capability); + } + } +} + +impl Serialize for Capabilities { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.iter()) + } +} + +impl<'de> Deserialize<'de> for Capabilities { + /// Skips any family string this build does not recognise, rather than + /// failing the whole deserialize. + /// + /// A remote driver speaking a newer minor contract version may advertise a + /// family this build has never heard of — see [`Capability::parse`] and the + /// module-level "wire stability" docs. Rejecting the whole handshake on one + /// unknown string would refuse an otherwise-compatible driver; the correct + /// behaviour is to drop the family this kernel could never call anyway. + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = Vec::::deserialize(deserializer)?; + let families = raw + .into_iter() + .filter_map(|family| Capability::parse(&family).ok()); + Ok(families.collect()) + } +} + +/// A driver advertised a capability set missing at least one mandatory family. +/// +/// Carries the missing families rather than a formatted string so the caller +/// can report them structurally (status RPC, bind-failure event) as well as in +/// a log line. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +#[error( + "memory driver advertises an incomplete capability set; missing mandatory families: {}", + .missing.iter().map(|c| c.as_str()).collect::>().join(", ") +)] +pub struct MissingMandatoryCapabilities { + /// Mandatory families absent from the advertised set, in + /// [`Capability::ALL`] order. Never empty. + pub missing: Vec, +} + +impl From for MemoryError { + /// An incomplete advertised set is a caller/config error, not an + /// unsupported call: the driver said something invalid about itself, which + /// is why this maps to [`MemoryError::Invalid`] and not + /// [`MemoryError::Unsupported`]. + fn from(value: MissingMandatoryCapabilities) -> Self { + MemoryError::Invalid(value.to_string()) + } +} + +#[cfg(test)] +#[path = "capabilities_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/capabilities_tests.rs b/src/openhuman/memory/api/capabilities_tests.rs new file mode 100644 index 0000000000..1e5e550edd --- /dev/null +++ b/src/openhuman/memory/api/capabilities_tests.rs @@ -0,0 +1,320 @@ +//! Unit tests for the capability vocabulary in [`super`]. +//! +//! Three properties are load-bearing and each has its own test: +//! +//! 1. the enum has exactly the thirteen contract families and no more; +//! 2. the serialized form is stable snake_case **strings**, never discriminant +//! integers — a driver deployed against an older build must keep advertising +//! the same set after a variant is inserted mid-enum; +//! 3. [`super::Capabilities::validate`] rejects a set missing **any** of the +//! three mandatory families, checked one family at a time. + +use super::*; +use serde_json::json; + +#[test] +fn capability_has_exactly_the_thirteen_contract_families() { + assert_eq!(Capability::ALL.len(), 13); + assert_eq!(Capability::all().len(), 13); + + let names: Vec<&str> = Capability::ALL.iter().map(|c| c.as_str()).collect(); + assert_eq!( + names, + vec![ + "core", + "recall", + "ingest", + "documents", + "tree", + "entities", + "graph", + "diff", + "goals", + "tool_memory", + "sources", + "maintenance", + "portability", + ] + ); +} + +#[test] +fn capability_all_has_no_duplicates() { + let mut seen = std::collections::BTreeSet::new(); + for capability in Capability::ALL { + assert!( + seen.insert(capability.as_str()), + "duplicate capability in ALL: {capability}" + ); + } +} + +#[test] +fn capability_as_str_matches_serde_representation() { + // The wire form is the stable contract; `as_str` is the authority and the + // derive must agree with it for every variant. + for capability in Capability::ALL { + assert_eq!( + serde_json::to_value(capability).unwrap(), + json!(capability.as_str()), + "serde form drifted from as_str for {capability}" + ); + } +} + +#[test] +fn capability_serializes_as_a_string_not_an_integer() { + // Guards the specific regression the string form exists to prevent: + // inserting a variant must not re-map an already-deployed driver's set. + for capability in Capability::ALL { + assert!( + serde_json::to_value(capability).unwrap().is_string(), + "{capability} did not serialize as a string" + ); + } +} + +#[test] +fn capability_parse_round_trips_every_variant() { + for capability in Capability::ALL { + assert_eq!(Capability::parse(capability.as_str()), Ok(capability)); + assert_eq!( + capability.as_str().parse::(), + Ok(capability), + "FromStr disagreed with parse for {capability}" + ); + let decoded: Capability = + serde_json::from_value(json!(capability.as_str())).expect("known family decodes"); + assert_eq!(decoded, capability); + } +} + +#[test] +fn capability_parse_rejects_unknown_family() { + let err = Capability::parse("quantum_recall").expect_err("unknown family must not parse"); + assert!(err.contains("quantum_recall"), "unhelpful error: {err}"); +} + +#[test] +fn mandatory_families_are_core_recall_and_portability() { + assert_eq!( + Capability::MANDATORY, + [ + Capability::Core, + Capability::Recall, + Capability::Portability + ] + ); + for capability in Capability::ALL { + assert_eq!( + capability.is_mandatory(), + matches!( + capability, + Capability::Core | Capability::Recall | Capability::Portability + ), + "wrong mandatory classification for {capability}" + ); + } +} + +#[test] +fn capabilities_all_contains_every_family() { + let all = Capabilities::all(); + assert_eq!(all.len(), Capability::ALL.len()); + for capability in Capability::ALL { + assert!(all.contains(capability), "all() is missing {capability}"); + } + assert!(!all.is_empty()); +} + +#[test] +fn capabilities_empty_contains_nothing() { + let none = Capabilities::empty(); + assert!(none.is_empty()); + assert_eq!(none.len(), 0); + for capability in Capability::ALL { + assert!(!none.contains(capability)); + } + // The default capability set is empty (the null driver itself advertises + // `Capabilities::mandatory()`, not the default). + assert_eq!(Capabilities::default(), none); +} + +#[test] +fn capabilities_bit_width_has_room_well_beyond_the_current_thirteen_families() { + // A `u16` bitset (the original representation) has exactly 16 bit + // positions, leaving room for only 3 more families before a family's + // `1 << index` bit-shift overflows. Pin the wider `u64` representation so + // a future family addition doesn't have to rediscover that ceiling. + assert!(std::mem::size_of::() * 8 >= 64); +} + +#[test] +fn capabilities_insert_and_remove_are_idempotent() { + let mut set = Capabilities::empty(); + set.insert(Capability::Tree); + set.insert(Capability::Tree); + assert_eq!(set.len(), 1); + assert!(set.contains(Capability::Tree)); + assert!(!set.contains(Capability::Graph)); + + set.remove(Capability::Tree); + set.remove(Capability::Tree); + assert!(set.is_empty()); +} + +#[test] +fn capabilities_builder_forms_mirror_insert_and_remove() { + let set = Capabilities::empty() + .with(Capability::Core) + .with(Capability::Recall) + .without(Capability::Recall); + assert!(set.contains(Capability::Core)); + assert!(!set.contains(Capability::Recall)); +} + +#[test] +fn capabilities_contains_all_checks_subsets() { + let full = Capabilities::all(); + let mandatory = Capabilities::mandatory(); + + assert!(full.contains_all(mandatory)); + assert!(!mandatory.contains_all(full)); + assert!(mandatory.contains_all(mandatory)); + assert!(full.contains_all(Capabilities::empty())); +} + +#[test] +fn capabilities_iterates_in_declaration_order() { + let set: Capabilities = [ + Capability::Portability, + Capability::Core, + Capability::Tree, + Capability::Recall, + ] + .into_iter() + .collect(); + + assert_eq!( + set.iter().collect::>(), + vec![ + Capability::Core, + Capability::Recall, + Capability::Tree, + Capability::Portability + ] + ); +} + +#[test] +fn capabilities_serde_round_trips_and_uses_a_string_array() { + let set = Capabilities::mandatory().with(Capability::ToolMemory); + let encoded = serde_json::to_value(set).unwrap(); + + // Declaration order, snake_case strings — the `capabilities[]` handshake + // field. + assert_eq!( + encoded, + json!(["core", "recall", "tool_memory", "portability"]) + ); + + let decoded: Capabilities = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded, set); +} + +#[test] +fn capabilities_full_set_serde_round_trips() { + let all = Capabilities::all(); + let encoded = serde_json::to_string(&all).unwrap(); + let decoded: Capabilities = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded, all); +} + +#[test] +fn capabilities_deserialization_collapses_duplicates_and_ignores_order() { + let decoded: Capabilities = + serde_json::from_value(json!(["portability", "core", "core", "recall"])).unwrap(); + assert_eq!(decoded, Capabilities::mandatory()); + assert_eq!(decoded.len(), 3); +} + +#[test] +fn capabilities_deserialization_skips_an_unknown_family() { + // A remote driver speaking a newer minor contract version may advertise a + // family this build has never heard of (see the module docs' "wire + // stability" section and `Capability::parse`). The handshake must still + // decode — with the unknown family dropped — rather than failing the bind + // outright. + let decoded: Capabilities = + serde_json::from_value(json!(["core", "warp_drive", "recall"])).unwrap(); + assert_eq!( + decoded, + Capabilities::empty() + .with(Capability::Core) + .with(Capability::Recall) + ); +} + +#[test] +fn validate_accepts_the_minimum_bindable_set() { + assert_eq!(Capabilities::mandatory().validate(), Ok(())); + assert_eq!(Capabilities::all().validate(), Ok(())); +} + +#[test] +fn validate_rejects_a_set_missing_core() { + let set = Capabilities::all().without(Capability::Core); + let err = set.validate().expect_err("missing core must be rejected"); + assert_eq!(err.missing, vec![Capability::Core]); + assert!(err.to_string().contains("core"), "{err}"); +} + +#[test] +fn validate_rejects_a_set_missing_recall() { + let set = Capabilities::all().without(Capability::Recall); + let err = set.validate().expect_err("missing recall must be rejected"); + assert_eq!(err.missing, vec![Capability::Recall]); + assert!(err.to_string().contains("recall"), "{err}"); +} + +#[test] +fn validate_rejects_a_set_missing_portability() { + // Portability is mandatory because without it a bind is a one-way door. + let set = Capabilities::all().without(Capability::Portability); + let err = set + .validate() + .expect_err("missing portability must be rejected"); + assert_eq!(err.missing, vec![Capability::Portability]); + assert!(err.to_string().contains("portability"), "{err}"); +} + +#[test] +fn validate_reports_every_missing_mandatory_family_at_once() { + let err = Capabilities::empty() + .validate() + .expect_err("the null set must be rejected"); + assert_eq!( + err.missing, + vec![ + Capability::Core, + Capability::Recall, + Capability::Portability + ] + ); +} + +#[test] +fn missing_mandatory_converts_to_an_invalid_memory_error() { + let err = Capabilities::empty().validate().unwrap_err(); + let message = err.to_string(); + let converted: MemoryError = err.into(); + // An incomplete advertised set is a bad claim about the driver, not an + // unsupported call. + assert!(matches!(converted, MemoryError::Invalid(ref m) if *m == message)); +} + +#[test] +fn missing_mandatory_is_empty_for_a_valid_set() { + assert!(Capabilities::mandatory().missing_mandatory().is_empty()); + assert!(Capabilities::all().missing_mandatory().is_empty()); +} diff --git a/src/openhuman/memory/api/chunks.rs b/src/openhuman/memory/api/chunks.rs new file mode 100644 index 0000000000..789bc1be2b --- /dev/null +++ b/src/openhuman/memory/api/chunks.rs @@ -0,0 +1,424 @@ +//! Core types for the memory chunk layer. +//! +//! This module defines the canonical [`Chunk`] representation produced by the +//! ingestion pipeline along with its provenance [`Metadata`] and back-pointer +//! [`SourceRef`]. +//! +//! All chunk IDs are deterministic: `sha256(source_kind | "\0" | source_id | +//! "\0" | seq | "\0" | content)` truncated to 32 hex chars so re-ingest of the +//! same source material yields stable IDs and idempotent upserts. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Which kind of upstream source produced a chunk. +/// +/// Used both as a metadata discriminator and as the routing key for the +/// canonicaliser dispatch in the ingest pipeline. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SourceKind { + /// Chat transcript scoped by channel or group (Slack, Discord, Telegram, WhatsApp…). + Chat, + /// Email thread (Gmail and generic IMAP). + Email, + /// Standalone document (Notion page, Drive doc, meeting note, uploaded file…). + Document, +} + +impl SourceKind { + /// Stable string representation for DB storage and RPC surfaces. + pub fn as_str(self) -> &'static str { + match self { + SourceKind::Chat => "chat", + SourceKind::Email => "email", + SourceKind::Document => "document", + } + } + + /// Parse back from the on-wire / on-disk string form. + /// + /// # Errors + /// + /// Returns an error when `s` is not a supported source kind. + pub fn parse(s: &str) -> Result { + match s { + "chat" => Ok(SourceKind::Chat), + "email" => Ok(SourceKind::Email), + "document" => Ok(SourceKind::Document), + other => Err(format!("unknown source kind: {other}")), + } + } +} + +/// Concrete upstream provider the content came from. +/// +/// Each variant maps to exactly one [`SourceKind`] via [`Self::kind`]. Wire +/// form is snake_case (see [`Self::as_str`] / [`Self::parse`]) so it is stable +/// across DB rows, JSON-RPC payloads, and logs. +/// +/// Marked `#[non_exhaustive]` so new providers can be added in later phases +/// without breaking downstream pattern matches. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[non_exhaustive] +pub enum DataSource { + // ── Chat transcripts (grouped by channel/group) ──────────────────── + /// Discord channel/server messages. Feeds [`SourceKind::Chat`]. + Discord, + /// Telegram chat/group messages. Feeds [`SourceKind::Chat`]. + Telegram, + /// WhatsApp chat/group messages. Feeds [`SourceKind::Chat`]. + Whatsapp, + + // ── Agent conversations (stored as durable memory) ──────────────── + /// Agent conversation transcripts persisted as durable memory. Feeds [`SourceKind::Chat`]. + Conversation, + + // ── Email threads (grouped by thread) ────────────────────────────── + /// Gmail thread. Feeds [`SourceKind::Email`]. + Gmail, + /// Catch-all for non-Gmail providers (Outlook, FastMail, generic IMAP, …). + OtherEmail, + + // ── Documents (no grouping) ──────────────────────────────────────── + /// Notion page. Feeds [`SourceKind::Document`]. + Notion, + /// Meeting notes document. Feeds [`SourceKind::Document`]. + MeetingNotes, + /// Google Drive document. Feeds [`SourceKind::Document`]. + DriveDocs, +} + +impl DataSource { + /// Which [`SourceKind`] this provider feeds into. + pub fn kind(self) -> SourceKind { + match self { + Self::Discord | Self::Telegram | Self::Whatsapp | Self::Conversation => { + SourceKind::Chat + } + Self::Gmail | Self::OtherEmail => SourceKind::Email, + Self::Notion | Self::MeetingNotes | Self::DriveDocs => SourceKind::Document, + } + } + + /// Stable snake_case identifier for DB storage, RPC payloads, and logs. + pub fn as_str(self) -> &'static str { + match self { + Self::Discord => "discord", + Self::Telegram => "telegram", + Self::Whatsapp => "whatsapp", + Self::Conversation => "conversation", + Self::Gmail => "gmail", + Self::OtherEmail => "other_email", + Self::Notion => "notion", + Self::MeetingNotes => "meeting_notes", + Self::DriveDocs => "drive_docs", + } + } + + /// Parse back from the on-wire / on-disk string form. + /// + /// # Errors + /// + /// Returns an error when `s` is not a supported data source. + pub fn parse(s: &str) -> Result { + match s { + "discord" => Ok(Self::Discord), + "telegram" => Ok(Self::Telegram), + "whatsapp" => Ok(Self::Whatsapp), + "conversation" => Ok(Self::Conversation), + "gmail" => Ok(Self::Gmail), + "other_email" => Ok(Self::OtherEmail), + "notion" => Ok(Self::Notion), + "meeting_notes" => Ok(Self::MeetingNotes), + "drive_docs" => Ok(Self::DriveDocs), + other => Err(format!("unknown data source: {other}")), + } + } + + /// Every known variant, in declaration order. Useful for tests, CLI + /// completion, and enumerating supported providers in diagnostic output. + pub fn all() -> &'static [DataSource] { + &[ + Self::Discord, + Self::Telegram, + Self::Whatsapp, + Self::Conversation, + Self::Gmail, + Self::OtherEmail, + Self::Notion, + Self::MeetingNotes, + Self::DriveDocs, + ] + } +} + +/// A concrete pointer back to where a chunk originated — used for citation, +/// drill-down, and deduplication at re-ingest time. +/// +/// Consumers should treat this as an opaque, source-specific reference. The +/// shape depends on [`SourceKind`]: +/// - **Chat**: `{platform}://{channel}/{message_id}` or `{permalink}` +/// - **Email**: message-id header (``) or provider URL +/// - **Document**: file path, Notion page URL, Drive file id +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct SourceRef { + /// Opaque provider-specific identifier for the exact source record. + pub value: String, +} + +impl SourceRef { + /// Wrap an opaque provider-specific identifier as a [`SourceRef`]. + pub fn new(value: impl Into) -> Self { + Self { + value: value.into(), + } + } +} + +/// Provenance metadata captured per chunk at ingest time. +/// +/// Captures at minimum: source type, source identifier, owner/account, +/// timestamps, and tags/labels when available. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Metadata { + /// Which upstream source kind produced this chunk. + pub source_kind: SourceKind, + /// Stable logical id for the ingestion group (channel id, thread id, doc id). + /// + /// Chat: channel/group id. Email: thread id. Document: doc id. + pub source_id: String, + /// Account or user the content belongs to. Empty string for anonymous / system sources. + pub owner: String, + /// Point-in-time timestamp for ordering within a source. + /// + /// For chats = message time; for emails = message sent time; + /// for documents = last-modified or ingest time. + #[serde(with = "chrono::serde::ts_milliseconds")] + pub timestamp: DateTime, + /// Covering time range the chunk spans. For a single leaf it usually equals + /// `(timestamp, timestamp)`; for later summary nodes it widens to cover all + /// children. + #[serde(with = "time_range_serde")] + pub time_range: (DateTime, DateTime), + /// Arbitrary labels / tags carried through from the source (e.g. Gmail labels, + /// Slack reactions, Notion tags). Ingest does not interpret these. + #[serde(default)] + pub tags: Vec, + /// Opaque pointer back to the raw source record for drill-down / citation. + pub source_ref: Option, + /// When set, overrides `source_id` for the chunk file path so multiple + /// items share one directory. `source_id` remains the dedup key. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path_scope: Option, +} + +impl Metadata { + /// Convenience constructor used by canonicalisers: point timestamp, + /// `time_range = (timestamp, timestamp)`. + pub fn point_in_time( + source_kind: SourceKind, + source_id: impl Into, + owner: impl Into, + timestamp: DateTime, + ) -> Self { + Self { + source_kind, + source_id: source_id.into(), + owner: owner.into(), + timestamp, + time_range: (timestamp, timestamp), + tags: Vec::new(), + source_ref: None, + path_scope: None, + } + } +} + +/// A single ingested chunk — the atomic persistence unit. +/// +/// In the design this is the leaf of a source tree. Later phases build summary +/// nodes on top of these leaves; here they live standalone. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Chunk { + /// Deterministic id derived from (source_kind, source_id, seq_in_source, content). + pub id: String, + /// Canonical Markdown content. + pub content: String, + /// Provenance metadata. + pub metadata: Metadata, + /// Token count (rough heuristic — 1 token ≈ 4 chars). + pub token_count: u32, + /// Sequence number of this chunk inside its logical source. Stable and + /// starts at 0 for the first chunk of a source. + pub seq_in_source: u32, + /// When this chunk was persisted to the local store. + #[serde(with = "chrono::serde::ts_milliseconds")] + pub created_at: DateTime, + /// True when this chunk is a sub-split of a single logical unit (e.g. a + /// chat message or email body that exceeded `max_tokens`). Each piece + /// carries this flag so downstream scorers can lower its weight relative to + /// whole-unit chunks. + #[serde(default)] + pub partial_message: bool, +} + +/// A chunk staged for the MD-content write path: a [`Chunk`] whose full body +/// lives on disk at `content_path` (with `content_sha256` for integrity), while +/// the SQLite `content` column carries only a ≤500-char preview. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StagedChunk { + /// The chunk being persisted. + pub chunk: Chunk, + /// Forward-slash relative path (under the content root) where the full body lives. + pub content_path: String, + /// Hex SHA-256 of the on-disk body, recorded for integrity checks. + pub content_sha256: String, +} + +/// Deterministic chunk id. +/// +/// `sha256(source_kind | "\0" | source_id | "\0" | seq | "\0" | content)` +/// hex-encoded, first 32 chars (128 bits of collision resistance). +/// +/// Content is included so multiple ingest calls that share a `source_id` don't +/// collide on `seq=0,1,2,…`. Re-ingesting the same canonical content under the +/// same `(source_id, seq)` still produces the same id, so upserts stay +/// idempotent. +pub fn chunk_id( + source_kind: SourceKind, + source_id: &str, + seq_in_source: u32, + content: &str, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(source_kind.as_str().as_bytes()); + hasher.update([0u8]); + hasher.update(source_id.as_bytes()); + hasher.update([0u8]); + hasher.update(seq_in_source.to_be_bytes()); + hasher.update([0u8]); + hasher.update(content.as_bytes()); + let digest = hasher.finalize(); + let hex = digest.iter().fold(String::with_capacity(64), |mut acc, b| { + use std::fmt::Write; + let _ = write!(acc, "{b:02x}"); + acc + }); + hex[..32].to_string() +} + +/// Approximate token count (GPT-family heuristic: 1 token ≈ 4 chars). +pub fn approx_token_count(text: &str) -> u32 { + // saturating_add guards against absurdly long inputs + let chars = text.chars().count() as u32; + chars.saturating_add(3) / 4 +} + +/// Per-character weight in **quarter-token** units for +/// [`conservative_token_estimate`]. Deliberately pessimistic so the chunker and +/// the embed backstop never under-split: real SentencePiece/WordPiece output for +/// hash-, code-, and markdown-dense text approaches ~1 token/char — far above +/// the `chars/4` GPT heuristic in [`approx_token_count`]. +fn char_token_quarters(ch: char) -> u32 { + if ch.is_ascii_alphanumeric() { + 2 // 0.50 token/char — alphanumeric runs pack ~2-4 chars per token + } else if ch.is_whitespace() { + 1 // 0.25 token/char — whitespace usually merges into adjacent pieces + } else { + 4 // 1.00 token/char — ASCII punctuation/symbols AND all non-ASCII + // (Hebrew/CJK/emoji), which tokenise ~1 piece per char or worse + } +} + +/// Conservative (over-estimating) token count, for embed-safety decisions only. +/// +/// [`approx_token_count`] (`chars/4`) under-counts dense markdown/hash/code by +/// ~5×. This weights characters by class so the result is an upper-ish bound on +/// real tokeniser output. It does **not** replace `approx_token_count`, which +/// still drives summariser/seal token budgeting. +pub fn conservative_token_estimate(text: &str) -> u32 { + let quarters: u64 = text + .chars() + .map(|c| u64::from(char_token_quarters(c))) + .sum(); + let tokens = quarters.div_ceil(4); // ceil(quarters / 4) + tokens.min(u64::from(u32::MAX)) as u32 +} + +/// Largest leading slice of `text` whose [`conservative_token_estimate`] is +/// ≤ `budget`, ending on a UTF-8 char boundary. Returns the whole string when +/// already within budget. Used as the embed-path backstop so an over-long body +/// can never be sent to the embedder above its input limit. +pub fn truncate_to_conservative_tokens(text: &str, budget: u32) -> &str { + if conservative_token_estimate(text) <= budget { + return text; + } + let cap = u64::from(budget).saturating_mul(4); // quarter-tokens + let mut acc: u64 = 0; + for (idx, ch) in text.char_indices() { + let q = u64::from(char_token_quarters(ch)); + if acc + q > cap { + return &text[..idx]; + } + acc += q; + } + text +} + +/// `serde(with = ...)` shim for `(DateTime, DateTime)`. +/// +/// Chrono has no built-in serde helper for a *pair* of timestamps, so this +/// mirrors `chrono::serde::ts_milliseconds` but for a 2-tuple: each endpoint +/// round-trips through millisecond-since-epoch integers under the field +/// names `start_ms` / `end_ms`. +mod time_range_serde { + use chrono::{DateTime, TimeZone, Utc}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + /// On-wire shape: millisecond-since-epoch pair. + #[derive(Serialize, Deserialize)] + struct Wire { + start_ms: i64, + end_ms: i64, + } + + /// Serialize a `(start, end)` UTC timestamp pair as `{start_ms, end_ms}`. + pub fn serialize( + value: &(DateTime, DateTime), + serializer: S, + ) -> Result { + Wire { + start_ms: value.0.timestamp_millis(), + end_ms: value.1.timestamp_millis(), + } + .serialize(serializer) + } + + /// Deserialize a `{start_ms, end_ms}` pair back into UTC timestamps. + /// + /// # Errors + /// Returns a `serde` custom error if either millisecond value does not + /// map to a valid `DateTime` (chrono's `timestamp_millis_opt` fails, + /// e.g. out-of-range values). + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result<(DateTime, DateTime), D::Error> { + let wire = Wire::deserialize(deserializer)?; + let start = Utc + .timestamp_millis_opt(wire.start_ms) + .single() + .ok_or_else(|| serde::de::Error::custom("invalid start_ms"))?; + let end = Utc + .timestamp_millis_opt(wire.end_ms) + .single() + .ok_or_else(|| serde::de::Error::custom("invalid end_ms"))?; + Ok((start, end)) + } +} + +#[cfg(test)] +#[path = "chunks_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/chunks_tests.rs b/src/openhuman/memory/api/chunks_tests.rs new file mode 100644 index 0000000000..3d9c89f4dc --- /dev/null +++ b/src/openhuman/memory/api/chunks_tests.rs @@ -0,0 +1,209 @@ +//! Unit tests for the chunk model (`super`). + +use super::*; +use chrono::TimeZone; + +#[test] +fn chunk_id_is_deterministic() { + let a = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello"); + let b = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello"); + assert_eq!(a, b); + assert_eq!(a, "95785e45df3ff65599a71866e0412993"); + assert_eq!(a.len(), 32); +} + +#[test] +fn conservative_estimate_weights_by_char_class() { + assert_eq!(conservative_token_estimate("abcd"), 2); // 4 alnum × 2q / 4 + assert_eq!(conservative_token_estimate(" "), 1); // 4 ws × 1q / 4 + assert_eq!(conservative_token_estimate("....,,,,"), 8); // 8 punct × 4q / 4 + assert_eq!(conservative_token_estimate("שלום"), 4); // 4 non-ascii × 4q / 4 + assert_eq!(conservative_token_estimate(""), 0); +} + +#[test] +fn conservative_estimate_exceeds_approx_for_dense_content() { + let dense = "claude-memory:openhuman:MEMORY.md:67d6fe2727d431b16d41630babfdcf1cdf61bda7b9ba\n" + .repeat(40); + assert!( + conservative_token_estimate(&dense) > approx_token_count(&dense), + "conservative estimate must exceed chars/4 on dense content", + ); +} + +#[test] +fn truncate_respects_budget_and_char_boundaries() { + let text = "שלום עולם ".repeat(100); // Hebrew, ~1 token/char + let out = truncate_to_conservative_tokens(&text, 10); + assert!(conservative_token_estimate(out) <= 10); + assert!(text.starts_with(out)); // valid prefix on a char boundary + assert!(out.len() < text.len()); +} + +#[test] +fn truncate_is_noop_within_budget() { + let text = "short and sweet"; + assert_eq!(truncate_to_conservative_tokens(text, 1000), text); +} + +#[test] +fn chunk_id_varies_with_seq() { + let a = chunk_id(SourceKind::Chat, "slack:#eng", 0, "hello"); + let b = chunk_id(SourceKind::Chat, "slack:#eng", 1, "hello"); + assert_ne!(a, b); +} + +#[test] +fn chunk_id_varies_with_source_kind() { + let a = chunk_id(SourceKind::Chat, "foo", 0, "hello"); + let b = chunk_id(SourceKind::Email, "foo", 0, "hello"); + assert_ne!(a, b); +} + +#[test] +fn chunk_id_varies_with_source_id() { + let a = chunk_id(SourceKind::Chat, "x", 0, "hello"); + let b = chunk_id(SourceKind::Chat, "y", 0, "hello"); + assert_ne!(a, b); +} + +#[test] +fn chunk_id_varies_with_content() { + let a = chunk_id(SourceKind::Chat, "slack:c1", 0, "bucket A content"); + let b = chunk_id(SourceKind::Chat, "slack:c1", 0, "bucket B content"); + assert_ne!(a, b); +} + +#[test] +fn source_kind_round_trip() { + for kind in [SourceKind::Chat, SourceKind::Email, SourceKind::Document] { + assert_eq!(SourceKind::parse(kind.as_str()).unwrap(), kind); + } +} + +#[test] +fn data_source_round_trip() { + for ds in DataSource::all() { + assert_eq!(DataSource::parse(ds.as_str()).unwrap(), *ds); + } +} + +#[test] +fn data_source_has_all_variants() { + assert_eq!(DataSource::all().len(), 9); +} + +#[test] +fn data_source_kind_mapping() { + use DataSource::*; + for ds in [Discord, Telegram, Whatsapp, Conversation] { + assert_eq!(ds.kind(), SourceKind::Chat); + } + for ds in [Gmail, OtherEmail] { + assert_eq!(ds.kind(), SourceKind::Email); + } + for ds in [Notion, MeetingNotes, DriveDocs] { + assert_eq!(ds.kind(), SourceKind::Document); + } +} + +#[test] +fn data_source_parse_rejects_unknown() { + assert!(DataSource::parse("nope").is_err()); + assert!(DataSource::parse("Discord").is_err()); // case-sensitive + assert!(DataSource::parse("drive docs").is_err()); // no spaces +} + +#[test] +fn data_source_serde_is_snake_case() { + let ds = DataSource::MeetingNotes; + let json = serde_json::to_string(&ds).unwrap(); + assert_eq!(json, "\"meeting_notes\""); + let parsed: DataSource = serde_json::from_str("\"meeting_notes\"").unwrap(); + assert_eq!(parsed, ds); +} + +#[test] +fn approx_token_count_scales_linearly() { + assert_eq!(approx_token_count(""), 0); + assert_eq!(approx_token_count("a"), 1); // 1→1 + assert_eq!(approx_token_count("abcd"), 1); // 4→1 + assert_eq!(approx_token_count("abcde"), 2); // 5→2 + assert_eq!(approx_token_count(&"x".repeat(400)), 100); +} + +#[test] +fn source_kind_parse_rejects_unknown_wire_values() { + assert_eq!( + SourceKind::parse("video").unwrap_err(), + "unknown source kind: video" + ); +} + +#[test] +fn metadata_constructor_and_source_ref_fill_documented_defaults() { + let timestamp = Utc.timestamp_millis_opt(1_700_000_000_123).unwrap(); + let mut metadata = Metadata::point_in_time(SourceKind::Document, "doc-1", "alice", timestamp); + metadata.source_ref = Some(SourceRef::new("notion://doc-1")); + + assert_eq!(metadata.source_id, "doc-1"); + assert_eq!(metadata.owner, "alice"); + assert_eq!(metadata.time_range, (timestamp, timestamp)); + assert!(metadata.tags.is_empty()); + assert_eq!(metadata.source_ref.unwrap().value, "notion://doc-1"); +} + +#[test] +fn chunk_json_round_trips_millisecond_time_range_and_partial_default() { + let timestamp = Utc.timestamp_millis_opt(1_700_000_000_123).unwrap(); + let chunk = Chunk { + id: "chunk".into(), + content: "body".into(), + metadata: Metadata::point_in_time(SourceKind::Chat, "channel", "alice", timestamp), + token_count: 1, + seq_in_source: 0, + created_at: timestamp, + partial_message: true, + }; + let encoded = serde_json::to_value(&chunk).unwrap(); + assert_eq!( + encoded["metadata"]["time_range"]["start_ms"], + timestamp.timestamp_millis() + ); + assert_eq!(serde_json::from_value::(encoded).unwrap(), chunk); + + let mut legacy = serde_json::to_value(&chunk).unwrap(); + legacy.as_object_mut().unwrap().remove("partial_message"); + assert!( + !serde_json::from_value::(legacy) + .unwrap() + .partial_message + ); +} + +#[test] +fn chunk_json_rejects_out_of_range_time_range_endpoints() { + let timestamp = Utc.timestamp_millis_opt(1_700_000_000_123).unwrap(); + let chunk = Chunk { + id: "chunk".into(), + content: "body".into(), + metadata: Metadata::point_in_time(SourceKind::Chat, "channel", "alice", timestamp), + token_count: 1, + seq_in_source: 0, + created_at: timestamp, + partial_message: false, + }; + let mut encoded = serde_json::to_value(chunk).unwrap(); + encoded["metadata"]["time_range"]["start_ms"] = serde_json::json!(i64::MAX); + assert!(serde_json::from_value::(encoded.clone()) + .unwrap_err() + .to_string() + .contains("invalid start_ms")); + + encoded["metadata"]["time_range"]["start_ms"] = serde_json::json!(0); + encoded["metadata"]["time_range"]["end_ms"] = serde_json::json!(i64::MAX); + assert!(serde_json::from_value::(encoded) + .unwrap_err() + .to_string() + .contains("invalid end_ms")); +} diff --git a/src/openhuman/memory/api/error.rs b/src/openhuman/memory/api/error.rs new file mode 100644 index 0000000000..0c493f166c --- /dev/null +++ b/src/openhuman/memory/api/error.rs @@ -0,0 +1,104 @@ +//! Engine-level error type shared by ported modules that want a typed error +//! surface. Modules that mirror OpenHuman's `anyhow`-based signatures may keep +//! using `anyhow::Result`; this enum is for contracts that benefit from +//! matchable variants (validation, not-found, taint, IO). +//! +//! `?` converts `std::io::Error` and `serde_json::Error` into +//! [`MemoryError::Io`] / [`MemoryError::Serde`] automatically via the derived +//! `#[from]` impls, and any `anyhow::Error` (including one produced by `?` on +//! a foreign error type inside an `anyhow`-returning function) into +//! [`MemoryError::Other`]. The purpose-built variants ([`MemoryError::NotFound`], +//! [`MemoryError::Invalid`], [`MemoryError::BudgetExceeded`], +//! [`MemoryError::PathEscape`]) are constructed explicitly by callers that want +//! matchable, typed failure — they are never inferred from a foreign error. +//! +//! [`MemoryError::Unsupported`] is the one variant that belongs to the *driver +//! contract* rather than the engine: it is what a caller gets when a bound +//! driver does not implement the capability family a call needs. See its docs +//! for why that should be rare. + +use thiserror::Error; + +use crate::openhuman::memory::api::capabilities::Capability; + +/// Errors surfaced by the memory engine. +#[derive(Debug, Error)] +pub enum MemoryError { + /// A requested record / source / node was not found. + #[error("not found: {0}")] + NotFound(String), + /// Caller-supplied input failed validation. + #[error("invalid input: {0}")] + Invalid(String), + /// A configured budget (tokens, cost, depth) was exceeded. + #[error("budget exceeded: {0}")] + BudgetExceeded(String), + /// A path escaped the workspace sandbox (symlink / traversal). + #[error("path escapes workspace: {0}")] + PathEscape(String), + /// Underlying IO failure. + #[error("io error: {0}")] + Io(#[from] std::io::Error), + /// Serialization / deserialization failure. + #[error("serde error: {0}")] + Serde(#[from] serde_json::Error), + /// The bound driver does not implement the named capability family. + /// + /// This should be **rare**, because capabilities are negotiated once at + /// bind time and the kernel unregisters the RPC methods and omits the agent + /// tools of every unadvertised family. Reaching this variant means one of: + /// + /// - an out-of-process driver answered `501` for a family its handshake + /// claimed (the case [`crate::openhuman::memory::api::capabilities`] cannot pre-empt); + /// - a caller bypassed the capability filter — a kernel bug. + /// + /// ## Why the payload is an owned `String` and not a [`Capability`] + /// + /// The transport adapter constructs this from a wire response, where the + /// family is a runtime string that may not be a known [`Capability`] at all + /// — a driver speaking a newer minor contract version, a vendor extension, + /// or simply a typo in a third-party backend. A `Capability` field would + /// force the adapter to drop that information or fail parsing, and a + /// `&'static str` cannot be produced from a runtime value without leaking + /// memory. An owned `String` is the only representation that round-trips + /// every case. + /// + /// Construct it with [`MemoryError::unsupported`] when the family is known + /// (that path yields the canonical [`Capability::as_str`] spelling) and + /// with [`MemoryError::unsupported_raw`] when it came off the wire. + #[error("unsupported capability: {capability}")] + Unsupported { + /// Wire name of the capability family that is not supported — + /// [`Capability::as_str`] when known, otherwise the raw string the + /// driver reported. + capability: String, + }, + /// Catch-all wrapping an opaque lower-level error. + #[error(transparent)] + Other(#[from] anyhow::Error), +} + +impl MemoryError { + /// Builds [`MemoryError::Unsupported`] for a family this build knows, + /// using its canonical [`Capability::as_str`] spelling. + pub fn unsupported(capability: Capability) -> Self { + Self::Unsupported { + capability: capability.as_str().to_string(), + } + } + + /// Builds [`MemoryError::Unsupported`] from a family name that came off the + /// wire and may not correspond to any known [`Capability`]. + pub fn unsupported_raw(capability: impl Into) -> Self { + Self::Unsupported { + capability: capability.into(), + } + } +} + +/// Convenience result alias for engine-level fallible operations. +pub type MemoryEngineResult = Result; + +#[cfg(test)] +#[path = "error_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/error_tests.rs b/src/openhuman/memory/api/error_tests.rs new file mode 100644 index 0000000000..0a8d7ab06f --- /dev/null +++ b/src/openhuman/memory/api/error_tests.rs @@ -0,0 +1,59 @@ +//! Unit tests for [`super::MemoryError`], focused on the `Unsupported` variant +//! added for the driver contract. The older variants are exercised where they +//! are constructed, in the engine crate. + +use super::*; +use crate::openhuman::memory::api::capabilities::Capability; + +#[test] +fn unsupported_from_a_known_capability_uses_the_canonical_wire_name() { + for capability in Capability::ALL { + let err = MemoryError::unsupported(capability); + match err { + MemoryError::Unsupported { + capability: ref got, + } => { + assert_eq!(got, capability.as_str()); + } + other => panic!("expected Unsupported, got {other:?}"), + } + } +} + +#[test] +fn unsupported_raw_preserves_a_family_this_build_does_not_know() { + // The reason the payload is an owned `String`: a driver speaking a newer + // minor contract version can name a family that is not a `Capability` here, + // and the adapter must be able to report it verbatim. + let err = MemoryError::unsupported_raw("holographic_recall"); + match err { + MemoryError::Unsupported { ref capability } => { + assert_eq!(capability, "holographic_recall"); + assert!(Capability::parse(capability).is_err()); + } + other => panic!("expected Unsupported, got {other:?}"), + } +} + +#[test] +fn unsupported_display_names_the_capability() { + assert_eq!( + MemoryError::unsupported(Capability::Tree).to_string(), + "unsupported capability: tree" + ); + assert_eq!( + MemoryError::unsupported(Capability::ToolMemory).to_string(), + "unsupported capability: tool_memory" + ); +} + +#[test] +fn unsupported_is_distinguishable_from_the_other_variants() { + // A transport adapter maps `501` to `Unsupported` and everything else + // elsewhere, so the variant must not collide with `Invalid` / `NotFound`. + let unsupported = MemoryError::unsupported(Capability::Diff); + assert!(matches!(unsupported, MemoryError::Unsupported { .. })); + + let invalid = MemoryError::Invalid("diff".to_string()); + assert!(!matches!(invalid, MemoryError::Unsupported { .. })); +} diff --git a/src/openhuman/memory/api/goals.rs b/src/openhuman/memory/api/goals.rs new file mode 100644 index 0000000000..697a86784b --- /dev/null +++ b/src/openhuman/memory/api/goals.rs @@ -0,0 +1,131 @@ +//! Domain types for the agent's long-term goals list. +//! +//! Goals are a small, ordered list of durable objectives the agent holds when +//! interacting with the user. They are persisted as a compact markdown document +//! (`MEMORY_GOALS.md`) by the engine crate's `memory::goals::store` and +//! surfaced over RPC + agent tools. Each item carries a stable short id so +//! edit/delete operations can address a specific line without depending on +//! ordering. +//! +//! This module is **pure data**: it owns the shape, parse, and render only. +//! The validating mutation surface (`add` / `edit` / `delete`) lives next to +//! the `regex`-backed PII/secret predicates it calls, in the engine crate's +//! `memory::goals::store::GoalsDocMutations` trait, so the value types stay +//! free of the safety machinery and of `regex`. The cap-enforcing persistence +//! layer and the reflection apply/dedupe logic live in the engine crate too. + +use serde::{Deserialize, Serialize}; + +/// Markdown header rendered at the top of `MEMORY_GOALS.md`. +pub(crate) const HEADER: &str = "# Long-term Goals"; + +/// A single long-term goal item. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GoalItem { + /// Stable short id (e.g. `g1`). Used as the dedupe/address key for + /// `edit`/`delete`. Rendered inline in the markdown as `- [g1] …`. + pub id: String, + /// The goal text — one concise sentence. + pub text: String, +} + +impl GoalItem { + /// Construct a goal item from an id + text, trimming surrounding + /// whitespace from the text. + pub fn new(id: impl Into, text: impl Into) -> Self { + Self { + id: id.into(), + text: text.into().trim().to_string(), + } + } +} + +/// The full goals document — an ordered list of [`GoalItem`]s. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GoalsDoc { + /// Ordered goal items. Order is meaningful for rendering and cap trimming + /// (oldest = front). + pub items: Vec, +} + +impl GoalsDoc { + /// Parse a `MEMORY_GOALS.md` body into a [`GoalsDoc`]. + /// + /// Recognised item lines look like `- [g1] do the thing`. Lines that don't + /// match (the header, blank lines, free prose) are ignored so a + /// hand-edited file degrades gracefully rather than erroring. + pub fn parse(body: &str) -> Self { + let mut items = Vec::new(); + for line in body.lines() { + let trimmed = line.trim(); + // Strip the leading list marker, if present. + let rest = match trimmed.strip_prefix("- ") { + Some(r) => r.trim(), + None => continue, + }; + // Expect `[id] text`. + let Some(after_open) = rest.strip_prefix('[') else { + continue; + }; + let Some(close_idx) = after_open.find(']') else { + continue; + }; + let id = after_open[..close_idx].trim(); + let text = after_open[close_idx + 1..].trim(); + if id.is_empty() || text.is_empty() { + continue; + } + items.push(GoalItem::new(id, text)); + } + Self { items } + } + + /// Render the document back to markdown suitable for `MEMORY_GOALS.md`. + /// + /// NOTE: this emits only the header and the recognised `- [id] text` + /// item lines — any free prose, sub-bullets, or other hand-added content + /// a user wrote into the file is not represented in [`GoalsDoc`] and is + /// therefore dropped on the next `parse` → mutate → `render` round-trip + /// (e.g. via `add`/`edit`/`delete`/reflection). Treat this file as + /// machine-owned rather than freely hand-editable. + pub fn render(&self) -> String { + let mut out = String::from(HEADER); + out.push_str("\n\n"); + for item in &self.items { + out.push_str(&format!("- [{}] {}\n", item.id, item.text)); + } + out + } + + /// Whether the list currently has no items. Used to drive the + /// "first run / initial population" reflection behaviour. + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + /// Number of goal items currently held. + pub fn len(&self) -> usize { + self.items.len() + } + + /// Allocate the next free `g` id not already used in the list. + pub fn next_id(&self) -> String { + let mut n = self.items.len() + 1; + loop { + let candidate = format!("g{n}"); + if !self.items.iter().any(|i| i.id == candidate) { + return candidate; + } + n += 1; + } + } + + /// Whether the list already holds `id`. + pub fn contains_id(&self, id: &str) -> bool { + self.items.iter().any(|i| i.id == id) + } +} + +#[cfg(test)] +#[path = "goals_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/goals_tests.rs b/src/openhuman/memory/api/goals_tests.rs new file mode 100644 index 0000000000..e67410ce40 --- /dev/null +++ b/src/openhuman/memory/api/goals_tests.rs @@ -0,0 +1,22 @@ +//! Unit tests for [`super::GoalsDoc`] parse/render — the pure-data half. +//! +//! The validating mutation tests (`add` / `edit` / `delete`, including the +//! secret/PII rejection cases) live in the engine crate next to the +//! `GoalsDocMutations` trait that owns them: `memory::goals::mutations_tests`. + +use super::*; + +#[test] +fn render_starts_with_header() { + let doc = GoalsDoc::default(); + assert!(doc.render().starts_with("# Long-term Goals")); +} + +#[test] +fn parse_ignores_non_item_lines() { + let body = "# Long-term Goals\n\nsome stray prose\n- [g1] real goal\n- malformed line\n"; + let doc = GoalsDoc::parse(body); + assert_eq!(doc.items.len(), 1); + assert_eq!(doc.items[0].id, "g1"); + assert_eq!(doc.items[0].text, "real goal"); +} diff --git a/src/openhuman/memory/api/health.rs b/src/openhuman/memory/api/health.rs new file mode 100644 index 0000000000..e98c3242d1 --- /dev/null +++ b/src/openhuman/memory/api/health.rs @@ -0,0 +1,119 @@ +//! Liveness state a memory driver reports about itself. +//! +//! ## Why this lives in the contract crate and not in the host +//! +//! The OpenHuman kernel has (or will have) a *generic* subsystem-agnostic +//! `DriverHealth` shared by memory, inference, channels, and sandbox. This crate +//! cannot name that type: `tinymemory-api` is the contract a third-party driver +//! compiles against, and a driver must be able to depend on it without pulling +//! in the OpenHuman host — nor should the next subsystem cut over inherit +//! generic kernel vocabulary from a *memory* crate. +//! +//! So the contract carries its own [`MemoryHealth`], and the host's memory +//! adapter converts. The conversion is deliberately trivial and lossless: this +//! is a **small closed enum with a reason string**, shaped one-for-one against +//! the kernel's `Ready | Degraded { reason } | Down { reason }`, not a +//! free-form struct that would need field-by-field mapping and would drift. +//! Keep it that way — if a driver needs to report something richer, it belongs +//! in a driver-specific status payload, not here. +//! +//! ## Wire form +//! +//! Serializes as an internally-tagged object with a stable snake_case `status` +//! discriminant, which is also the shape of the transport adapter's +//! `GET /v1/health` → `{ status, reason }` response: +//! +//! ```json +//! { "status": "ready" } +//! { "status": "degraded", "reason": "vector index rebuilding" } +//! { "status": "down", "reason": "connection refused" } +//! ``` + +use serde::{Deserialize, Serialize}; + +/// Health of a bound memory driver, as the driver reports it. +/// +/// The three states are ordered by severity and mean different things to the +/// kernel: +/// +/// - [`MemoryHealth::Ready`] — serve traffic normally. +/// - [`MemoryHealth::Degraded`] — still serve traffic, but surface the reason +/// in status output; results may be incomplete or slow. +/// - [`MemoryHealth::Down`] — do not serve traffic; the bind should be surfaced +/// as failed and, per the fallback rule, the embedded default rebound. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum MemoryHealth { + /// The driver is reachable and serving requests normally. + Ready, + /// The driver is serving requests, but something is wrong and the caller + /// should surface it. Results may be incomplete, stale, or slow. + Degraded { + /// Operator-facing explanation. Must not contain credentials, tokens, + /// or user memory content — this string is logged and shown in status + /// output. + reason: String, + }, + /// The driver cannot serve requests at all. + Down { + /// Operator-facing explanation, subject to the same redaction rule as + /// [`MemoryHealth::Degraded::reason`]. + reason: String, + }, +} + +impl MemoryHealth { + /// Convenience constructor for [`MemoryHealth::Degraded`]. + pub fn degraded(reason: impl Into) -> Self { + Self::Degraded { + reason: reason.into(), + } + } + + /// Convenience constructor for [`MemoryHealth::Down`]. + pub fn down(reason: impl Into) -> Self { + Self::Down { + reason: reason.into(), + } + } + + /// Stable snake_case discriminant, matching the serialized `status` field. + pub fn as_str(&self) -> &'static str { + match self { + Self::Ready => "ready", + Self::Degraded { .. } => "degraded", + Self::Down { .. } => "down", + } + } + + /// The operator-facing reason, when there is one. `None` for + /// [`MemoryHealth::Ready`]. + pub fn reason(&self) -> Option<&str> { + match self { + Self::Ready => None, + Self::Degraded { reason } | Self::Down { reason } => Some(reason.as_str()), + } + } + + /// Whether the kernel should route traffic to this driver. + /// + /// True for [`MemoryHealth::Ready`] and [`MemoryHealth::Degraded`] — a + /// degraded driver is still the bound driver — and false for + /// [`MemoryHealth::Down`]. + pub fn is_usable(&self) -> bool { + !matches!(self, Self::Down { .. }) + } +} + +impl std::fmt::Display for MemoryHealth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.reason() { + Some(reason) => write!(f, "{}: {reason}", self.as_str()), + None => f.write_str(self.as_str()), + } + } +} + +#[cfg(test)] +#[path = "health_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/health_tests.rs b/src/openhuman/memory/api/health_tests.rs new file mode 100644 index 0000000000..a31c96516c --- /dev/null +++ b/src/openhuman/memory/api/health_tests.rs @@ -0,0 +1,90 @@ +//! Unit tests for [`super::MemoryHealth`]. +//! +//! These pin the two properties the host's memory adapter depends on: the +//! variant set is closed and small enough for a lossless `match` into the +//! kernel's generic `DriverHealth`, and the wire form carries a stable +//! `status` discriminant plus a `reason`. + +use super::*; +use serde_json::json; + +#[test] +fn ready_has_no_reason_and_is_usable() { + let health = MemoryHealth::Ready; + assert_eq!(health.as_str(), "ready"); + assert_eq!(health.reason(), None); + assert!(health.is_usable()); + assert_eq!(health.to_string(), "ready"); +} + +#[test] +fn degraded_carries_a_reason_and_is_still_usable() { + let health = MemoryHealth::degraded("vector index rebuilding"); + assert_eq!(health.as_str(), "degraded"); + assert_eq!(health.reason(), Some("vector index rebuilding")); + // A degraded driver is still the bound driver. + assert!(health.is_usable()); + assert_eq!(health.to_string(), "degraded: vector index rebuilding"); +} + +#[test] +fn down_carries_a_reason_and_is_not_usable() { + let health = MemoryHealth::down("connection refused"); + assert_eq!(health.as_str(), "down"); + assert_eq!(health.reason(), Some("connection refused")); + assert!(!health.is_usable()); + assert_eq!(health.to_string(), "down: connection refused"); +} + +#[test] +fn health_serializes_with_a_stable_status_discriminant() { + assert_eq!( + serde_json::to_value(MemoryHealth::Ready).unwrap(), + json!({ "status": "ready" }) + ); + assert_eq!( + serde_json::to_value(MemoryHealth::degraded("slow")).unwrap(), + json!({ "status": "degraded", "reason": "slow" }) + ); + assert_eq!( + serde_json::to_value(MemoryHealth::down("gone")).unwrap(), + json!({ "status": "down", "reason": "gone" }) + ); +} + +#[test] +fn health_round_trips_through_serde() { + for health in [ + MemoryHealth::Ready, + MemoryHealth::degraded("reindexing"), + MemoryHealth::down("auth expired"), + ] { + let encoded = serde_json::to_string(&health).unwrap(); + let decoded: MemoryHealth = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded, health); + } +} + +#[test] +fn health_constructors_match_their_variants() { + assert_eq!( + MemoryHealth::degraded("x"), + MemoryHealth::Degraded { + reason: "x".to_string() + } + ); + assert_eq!( + MemoryHealth::down("y"), + MemoryHealth::Down { + reason: "y".to_string() + } + ); +} + +#[test] +fn degraded_without_a_reason_is_rejected_on_the_wire() { + // `reason` is mandatory: a degraded/down driver that explains nothing is + // useless in status output, so the contract refuses to decode it. + assert!(serde_json::from_value::(json!({ "status": "degraded" })).is_err()); + assert!(serde_json::from_value::(json!({ "status": "down" })).is_err()); +} diff --git a/src/openhuman/memory/api/host/cloud_providers.rs b/src/openhuman/memory/api/host/cloud_providers.rs new file mode 100644 index 0000000000..dfdd6c3f73 --- /dev/null +++ b/src/openhuman/memory/api/host/cloud_providers.rs @@ -0,0 +1,855 @@ +//! Cloud provider credential schema. +//! +//! Each entry in `Config::cloud_providers` represents one configured LLM +//! backend. Providers are keyed by a user-chosen `slug` (e.g. `"openai"`, +//! `"my-deepseek"`). The factory in `inference::provider::factory` +//! resolves workload-to-provider strings against this list at runtime using +//! the grammar `":"`. +//! +//! Legacy configs that use `type`/`default_model` are migrated in-memory on +//! load via `migrate_legacy_fields()`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BuiltinCloudProvider { + pub slug: &'static str, + pub label: &'static str, + pub endpoint: &'static str, + pub auth_style: AuthStyle, +} + +pub const BUILTIN_CLOUD_PROVIDERS: &[BuiltinCloudProvider] = &[ + BuiltinCloudProvider { + slug: "openhuman", + label: "OpenHuman", + endpoint: "https://api.openhuman.ai/v1", + auth_style: AuthStyle::OpenhumanJwt, + }, + BuiltinCloudProvider { + slug: "openai", + label: "OpenAI", + endpoint: "https://api.openai.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "anthropic", + label: "Anthropic", + endpoint: "https://api.anthropic.com/v1", + auth_style: AuthStyle::Anthropic, + }, + BuiltinCloudProvider { + slug: "openrouter", + label: "OpenRouter", + endpoint: "https://openrouter.ai/api/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "orcarouter", + label: "OrcaRouter", + endpoint: "https://api.orcarouter.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "gmi", + label: "GMI", + endpoint: "https://api.gmi-serving.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "fireworks", + label: "Fireworks", + endpoint: "https://api.fireworks.ai/inference/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "moonshot", + label: "Kimi (Moonshot)", + endpoint: "https://api.moonshot.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "groq", + label: "Groq", + endpoint: "https://api.groq.com/openai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "mistral", + label: "Mistral", + endpoint: "https://api.mistral.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "deepseek", + label: "DeepSeek", + endpoint: "https://api.deepseek.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "together", + label: "Together AI", + endpoint: "https://api.together.xyz/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "google", + label: "Google Gemini", + endpoint: "https://generativelanguage.googleapis.com/v1beta/openai", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "cerebras", + label: "Cerebras", + endpoint: "https://api.cerebras.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "xai", + label: "xAI", + endpoint: "https://api.x.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "huggingface", + label: "Hugging Face", + endpoint: "https://router.huggingface.co/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "nvidia", + label: "NVIDIA", + endpoint: "https://integrate.api.nvidia.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "zai", + label: "Z.AI", + endpoint: "https://api.z.ai/api/paas/v4", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "minimax", + label: "MiniMax", + // MiniMax exposes a full OpenAI-compatible surface at `/v1` + // (`/v1/chat/completions`, `/v1/models`). The previous `/anthropic` + // base + Anthropic auth pointed at MiniMax's Messages-protocol API, + // which OpenHuman does not speak — it only builds OpenAI-style + // `/chat/completions` and `/models` — so both chat and model-listing + // 404'd (`/anthropic/chat/completions`, `/anthropic/models`). The + // 404 on model-listing was Sentry TAURI-RUST-8X3. Use the `/v1` + // OpenAI surface with Bearer auth so both paths resolve. + endpoint: "https://api.minimax.io/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "stepfun", + label: "StepFun", + endpoint: "https://api.stepfun.ai/step_plan/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "kilocode", + label: "Kilo Code", + endpoint: "https://api.kilo.ai/api/gateway", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "deepinfra", + label: "DeepInfra", + endpoint: "https://api.deepinfra.com/v1/openai", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "novita", + label: "Novita", + endpoint: "https://api.novita.ai/v3/openai", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "venice", + label: "Venice", + endpoint: "https://api.venice.ai/api/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "vercel-ai-gateway", + label: "Vercel AI Gateway", + endpoint: "https://ai-gateway.vercel.sh/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "sumopod", + label: "SumoPod", + endpoint: "https://ai.sumopod.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "modelscope", + label: "ModelScope", + endpoint: "https://api-inference.modelscope.cn/v1", + auth_style: AuthStyle::Bearer, + }, +]; + +fn builtin_cloud_provider(type_str: &str) -> Option<&'static BuiltinCloudProvider> { + BUILTIN_CLOUD_PROVIDERS + .iter() + .find(|provider| provider.slug == type_str) +} + +/// Whether `slug` matches a built-in cloud provider preset. +/// +/// The chat factory uses this to decide capability defaults (e.g. whether the +/// provider exposes the OpenAI Responses API) only for providers we ship and +/// therefore know the API surface of. Custom / user-defined slugs are treated +/// as unknown and keep the permissive defaults. +pub fn is_builtin_cloud_slug(slug: &str) -> bool { + builtin_cloud_provider(slug).is_some() +} + +/// Whether a built-in cloud provider exposes the OpenAI **Responses API** +/// (`/v1/responses`). +/// +/// Only OpenAI's first-party endpoint serves `/responses`; every other built-in +/// preset (DeepSeek, Groq, Mistral, Fireworks, …) is chat-completions-only. +/// Enabling the chat-completions-404 → `/responses` fallback for those +/// guarantees a second 404 against an endpoint that does not exist, which floods +/// Sentry with an empty-body `" Responses API error:"` event +/// (TAURI-RUST-5EN — same class as the local-provider TAURI-RUST-59Y fix). The +/// factory consults this to build chat-completions-only built-ins with +/// `new_no_responses_fallback`. +/// +/// Custom / unknown slugs are intentionally NOT covered here (see +/// [`is_builtin_cloud_slug`]): a user-defined OpenAI-compatible endpoint may be +/// a genuine OpenAI proxy that does support `/responses`, so the factory keeps +/// the fallback for those. +pub fn builtin_cloud_supports_responses_api(slug: &str) -> bool { + matches!(slug, "openai") +} + +/// Extract the lowercased authority host from an endpoint URL, dropping the +/// scheme, any userinfo, the port, and the path. Returns `None` when no host +/// can be parsed. Tolerant of a missing scheme and of IPv6 literals. +pub fn endpoint_host(endpoint: &str) -> Option { + let s = endpoint.trim(); + // Drop the scheme (`https://…`); tolerate a bare `host/path` form. + let after_scheme = s.split_once("://").map(|(_, rest)| rest).unwrap_or(s); + // The authority ends at the first path / query / fragment delimiter. + let authority = after_scheme + .split(['/', '?', '#']) + .next() + .unwrap_or(after_scheme); + // Strip any `user:pass@` userinfo prefix. + let host_port = authority + .rsplit_once('@') + .map(|(_, host)| host) + .unwrap_or(authority); + // Strip the port, handling bracketed IPv6 literals (`[::1]:8080`). + let host = if let Some(rest) = host_port.strip_prefix('[') { + rest.split_once(']').map(|(h, _)| h).unwrap_or(rest) + } else { + host_port + .rsplit_once(':') + .map(|(h, _)| h) + .unwrap_or(host_port) + }; + let host = host.trim().to_ascii_lowercase(); + (!host.is_empty()).then_some(host) +} + +/// Whether `host` is the authority host of any built-in cloud **inference** +/// provider (e.g. `openrouter.ai`, `api.openai.com`, `api.groq.com`). +/// +/// Derived entirely from [`BUILTIN_CLOUD_PROVIDERS`] so the set stays in sync +/// with the provider registry. `host` is compared case-insensitively against +/// each preset's [`endpoint_host`]. +/// +/// # Why this exists +/// +/// `config.api_url` is overloaded: it is the chat/inference endpoint, but +/// `api::config::effective_backend_api_url` also reuses it as the +/// OpenHuman **backend** base for team/billing/auth calls. A BYO user who +/// points `api_url` at a provider's canonical base (`https://openrouter.ai/api/v1`) +/// would otherwise have every backend domain call routed to the inference host +/// → 400/404 (TAURI-RUST-HW1: 4932 `GET /teams/me/usage` 400s from `openrouter.ai`). +/// The backend-URL resolver uses this to treat such hosts as non-backend and +/// fall back to the default backend chain — the cloud analogue of the local-AI +/// guard that fixed the Ollama case (OPENHUMAN-TAURI-51/-80/-7Z). +pub fn host_is_builtin_cloud_provider(host: &str) -> bool { + let host = host.trim().to_ascii_lowercase(); + if host.is_empty() { + return false; + } + BUILTIN_CLOUD_PROVIDERS + .iter() + .any(|p| endpoint_host(p.endpoint).as_deref() == Some(host.as_str())) +} + +/// Whether an endpoint **host** is a known cloud host that does NOT serve the +/// OpenAI Responses API (`/v1/responses`) — i.e. it is chat-completions-only, +/// regardless of which user slug points at it. +/// +/// Derived entirely from [`BUILTIN_CLOUD_PROVIDERS`]: a host is chat-only when +/// some built-in preset uses it AND no preset at that host advertises the +/// Responses API (only OpenAI's `api.openai.com` does). This closes the +/// custom-slug gap behind the builtin-slug gate +/// ([`builtin_cloud_supports_responses_api`]): a user slug pointed at, e.g., +/// `integrate.api.nvidia.com` must never attempt `/responses` (TAURI-RUST-5A1), +/// while a genuinely unknown proxy host keeps the permissive fallback so a real +/// OpenAI proxy still gets `/responses`. +pub fn endpoint_host_is_chat_completions_only(endpoint: &str) -> bool { + let Some(host) = endpoint_host(endpoint) else { + return false; + }; + let mut matched_chat_only = false; + for provider in BUILTIN_CLOUD_PROVIDERS { + if endpoint_host(provider.endpoint).as_deref() == Some(host.as_str()) { + if builtin_cloud_supports_responses_api(provider.slug) { + // A Responses-capable built-in lives at this host → not chat-only. + return false; + } + matched_chat_only = true; + } + } + matched_chat_only +} + +/// Authentication header style for a cloud provider. +/// +/// Wire format is lowercase (e.g. `"bearer"`). Determines which HTTP headers +/// are attached when calling the provider's API. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum AuthStyle { + /// OpenAI-compatible: `Authorization: Bearer ` + #[default] + Bearer, + /// Anthropic: `x-api-key: ` + `anthropic-version: 2023-06-01` + Anthropic, + /// OpenHuman session JWT (injected by the backend provider, not stored here). + OpenhumanJwt, + /// No auth header — e.g. local Ollama. + None, +} + +impl AuthStyle { + pub fn as_str(&self) -> &'static str { + match self { + Self::Bearer => "bearer", + Self::Anthropic => "anthropic", + Self::OpenhumanJwt => "openhuman_jwt", + Self::None => "none", + } + } +} + +/// Endpoint config for one cloud LLM provider. +/// +/// **Note on secrets**: API keys are NOT stored on this struct. They live in +/// `auth-profiles.json` via `security::credentials::AuthService`, +/// keyed by `provider:` (falling back to bare `` for legacy +/// entries). The factory looks up the token at call time via +/// `inference::provider::factory::auth_key_for_slug`. +/// +/// ## Back-compat +/// +/// Old configs may have `type` and `default_model` fields. These are +/// tolerated on read (via `legacy_type` / `default_model`) but never written. +/// Call `migrate_legacy_fields()` after deserialising. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(default)] +pub struct CloudProviderCreds { + /// Opaque stable id, e.g. `"p_openai_a8c3f"`. Never shown in the UI. + /// Generated once by [`generate_provider_id`] and never changes. + pub id: String, + /// Routing key chosen by the user or seeded from the legacy type. + /// Lower-case alphanumeric + `-`. Must be unique per config and not in the + /// reserved list (see [`is_slug_reserved`]). The factory resolves + /// `":"` strings against this field. + pub slug: String, + /// Human-readable display label, supplied by the frontend. Not used in routing. + pub label: String, + /// OpenAI-compatible base URL (`/models`, `/chat/completions` etc. are appended). + pub endpoint: String, + /// Authentication header style. + pub auth_style: AuthStyle, + + // ── Back-compat: old `type` field ─────────────────────────────────────── + /// Legacy discriminator written by older builds. Read-only; never emitted. + #[serde(rename = "type", default, skip_serializing)] + pub legacy_type: Option, + + // ── Back-compat: old `default_model` field ────────────────────────────── + /// Legacy default model written by older builds. Read-only; never emitted. + #[serde(default, skip_serializing)] + pub default_model: Option, +} + +impl Default for CloudProviderCreds { + fn default() -> Self { + Self { + id: String::new(), + slug: String::new(), + label: String::new(), + endpoint: String::new(), + auth_style: AuthStyle::Bearer, + legacy_type: None, + default_model: None, + } + } +} + +/// Reserved slugs that may not be used for user-configured providers. +/// These are sentinels in the factory's routing grammar. +/// +/// `ollama` is deliberately NOT reserved: the AI settings panel registers an +/// `ollama` `cloud_providers` entry so `list_configured_models` can resolve +/// the user's chosen base_url for the model dropdown. The factory's chat +/// routing is unaffected — the `ollama:` prefix branch in +/// `factory::create_chat_provider_from_string` fires before the +/// `:` cloud-provider lookup, so a synthetic `ollama` entry +/// never reaches `make_cloud_provider_by_slug`. When no `cloud_providers` +/// row exists (config drift, upgrade from a build that only persisted +/// `config.local_ai.base_url`, flush-vs-probe race), +/// `inference::provider::ops::list_configured_models` +/// falls back to a synthetic entry via `synthesize_local_runtime_entry` +/// (Sentry TAURI-RUST-28Z fix). The same fallback applies to `lmstudio`. +pub fn is_slug_reserved(s: &str) -> bool { + matches!(s.trim(), "" | "cloud" | "openhuman" | "pid") +} + +/// Apply legacy field migration in-place. +/// +/// Idempotent: only fills in empty fields from the legacy `type`/`default_model` +/// values. Safe to call on already-migrated entries. +pub fn migrate_legacy_fields(entry: &mut CloudProviderCreds) { + let legacy_type = entry.legacy_type.clone().unwrap_or_default(); + let lt = legacy_type.trim(); + + // Slug from legacy type when missing. + if entry.slug.is_empty() && !lt.is_empty() { + entry.slug = lt.to_string(); + log::debug!( + "[config][cloud_providers] migrated slug from legacy type='{}' id={}", + lt, + entry.id + ); + } + + // Label from static map when missing. + if entry.label.is_empty() { + entry.label = legacy_label_for(if entry.slug.is_empty() { + lt + } else { + &entry.slug + }) + .to_string(); + log::debug!( + "[config][cloud_providers] migrated label='{}' for slug='{}' id={}", + entry.label, + entry.slug, + entry.id + ); + } + + // Endpoint from legacy defaults when missing. + if entry.endpoint.is_empty() { + let ep = legacy_default_endpoint(lt); + if !ep.is_empty() { + entry.endpoint = ep.to_string(); + } + } + + // Auth style from legacy type when still at default Bearer. + if entry.auth_style == AuthStyle::Bearer { + if let Some(provider) = builtin_cloud_provider(lt) { + entry.auth_style = provider.auth_style; + } + } +} + +/// Map a legacy type string (or slug) to a human-readable label. +fn legacy_label_for(type_str: &str) -> &'static str { + builtin_cloud_provider(type_str) + .map(|provider| provider.label) + .unwrap_or("Custom") +} + +/// Map a legacy type string to its well-known default endpoint. +fn legacy_default_endpoint(type_str: &str) -> &'static str { + builtin_cloud_provider(type_str) + .map(|provider| provider.endpoint) + .unwrap_or("") +} + +/// Generate a short opaque id for a new provider entry. +/// +/// Format: `"p__<5 random alphanumerics>"`, e.g. `"p_openai_a8c3f"`. +/// The random suffix is not cryptographically strong — it only needs to be +/// unique within a single user's config file. +pub fn generate_provider_id(slug: &str) -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + // Cheap pseudo-random from timestamp nanoseconds — adequate for local + // config uniqueness without pulling in a PRNG crate. + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .subsec_nanos(); + let chars: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; + let mut suffix = String::with_capacity(5); + let mut seed = nanos as usize; + for _ in 0..5 { + suffix.push(chars[seed % chars.len()] as char); + seed = seed + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + seed = (seed >> 33) ^ seed; + } + // Sanitise slug to only alphanumeric + '-' for the id prefix. + let safe_slug: String = slug + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' { + c + } else { + '_' + } + }) + .take(20) + .collect(); + format!("p_{}_{}", safe_slug, suffix) +} + +// ── Back-compat type alias ────────────────────────────────────────────────── +// Kept so existing code that imports `CloudProviderType` compiles without +// sweeping changes. New code should use `AuthStyle` directly. + +/// Legacy discriminator enum. **Deprecated**: use `AuthStyle` on new entries. +/// Retained only to satisfy callers that still pattern-match on +/// `CloudProviderType` (e.g. the migration module). Will be removed once all +/// call sites are updated to slug-keyed lookups. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum CloudProviderType { + Openhuman, + Openai, + Anthropic, + Openrouter, + Orcarouter, + Custom, +} + +impl CloudProviderType { + /// Well-known default base URL for each provider type. + pub fn default_endpoint(&self) -> &'static str { + match self { + Self::Openhuman => "https://api.openhuman.ai/v1", + Self::Openai => "https://api.openai.com/v1", + Self::Anthropic => "https://api.anthropic.com/v1", + Self::Openrouter => "https://openrouter.ai/api/v1", + Self::Orcarouter => "https://api.orcarouter.ai/v1", + Self::Custom => "", + } + } + + /// Human-readable label used in logs and error messages. + pub fn label(&self) -> &'static str { + match self { + Self::Openhuman => "OpenHuman", + Self::Openai => "OpenAI", + Self::Anthropic => "Anthropic", + Self::Openrouter => "OpenRouter", + Self::Orcarouter => "OrcaRouter", + Self::Custom => "Custom", + } + } + + /// Lowercase wire-format string (matches JSON serialisation). + pub fn as_str(&self) -> &'static str { + match self { + Self::Openhuman => "openhuman", + Self::Openai => "openai", + Self::Anthropic => "anthropic", + Self::Openrouter => "openrouter", + Self::Orcarouter => "orcarouter", + Self::Custom => "custom", + } + } + + /// Corresponding `AuthStyle`. + pub fn auth_style(&self) -> AuthStyle { + match self { + Self::Openhuman => AuthStyle::OpenhumanJwt, + Self::Anthropic => AuthStyle::Anthropic, + _ => AuthStyle::Bearer, + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + builtin_cloud_supports_responses_api, endpoint_host, + endpoint_host_is_chat_completions_only, host_is_builtin_cloud_provider, + is_builtin_cloud_slug, is_slug_reserved, migrate_legacy_fields, AuthStyle, + CloudProviderCreds, BUILTIN_CLOUD_PROVIDERS, + }; + + #[test] + fn reserved_slugs() { + for s in ["", " ", "cloud", "openhuman", "pid"] { + assert!(is_slug_reserved(s), "{s:?} must stay reserved"); + } + } + + // Regression: `ollama` was previously reserved, which made the AI settings + // panel unable to persist an `ollama` cloud_providers entry — so the + // model-list dropdown failed with "no cloud provider with id or slug + // 'ollama' found". The factory's chat routing is unaffected by this + // change because the `ollama:` prefix branch fires before any + // cloud_providers lookup. + #[test] + fn ollama_and_lmstudio_are_not_reserved() { + assert!( + !is_slug_reserved("ollama"), + "ollama must be usable as a cloud_providers slug for the /models probe" + ); + assert!( + !is_slug_reserved("lmstudio"), + "lmstudio is a free-form OpenAI-compatible slug" + ); + } + + #[test] + fn builtin_cloud_provider_defaults_cover_phase_one_presets() { + for (slug, label, endpoint, auth_style) in [ + ( + "groq", + "Groq", + "https://api.groq.com/openai/v1", + AuthStyle::Bearer, + ), + ( + "deepseek", + "DeepSeek", + "https://api.deepseek.com/v1", + AuthStyle::Bearer, + ), + ( + "minimax", + "MiniMax", + "https://api.minimax.io/v1", + AuthStyle::Bearer, + ), + ( + "sumopod", + "SumoPod", + "https://ai.sumopod.com/v1", + AuthStyle::Bearer, + ), + ( + "modelscope", + "ModelScope", + "https://api-inference.modelscope.cn/v1", + AuthStyle::Bearer, + ), + ] { + let mut entry = CloudProviderCreds { + id: format!("p_{slug}"), + legacy_type: Some(slug.to_string()), + ..Default::default() + }; + migrate_legacy_fields(&mut entry); + + assert_eq!(entry.slug, slug); + assert_eq!(entry.label, label); + assert_eq!(entry.endpoint, endpoint); + assert_eq!(entry.auth_style, auth_style); + } + } + + #[test] + fn builtin_cloud_provider_slugs_are_unique() { + let mut slugs = std::collections::HashSet::new(); + for provider in BUILTIN_CLOUD_PROVIDERS { + assert!( + slugs.insert(provider.slug), + "duplicate built-in cloud provider slug {}", + provider.slug + ); + } + } + + #[test] + fn is_builtin_cloud_slug_matches_presets_only() { + for slug in ["openai", "deepseek", "groq", "mistral"] { + assert!(is_builtin_cloud_slug(slug), "{slug} is a built-in preset"); + } + for slug in ["my-proxy", "custom-openai", "totally-unknown", ""] { + assert!( + !is_builtin_cloud_slug(slug), + "{slug:?} is not a built-in preset" + ); + } + } + + #[test] + fn only_openai_builtin_exposes_responses_api() { + assert!(builtin_cloud_supports_responses_api("openai")); + for slug in ["deepseek", "groq", "mistral", "fireworks", "together"] { + assert!( + !builtin_cloud_supports_responses_api(slug), + "{slug} is chat-completions-only and must not advertise the Responses API" + ); + } + } + + /// Drift guard (TAURI-RUST-5EN): couple the capability helper to the + /// preset list so adding a new built-in that wrongly claims the Responses + /// API — or renaming `openai` — fails CI rather than silently re-enabling + /// the guaranteed-404 `/responses` fallback. OpenAI's first-party endpoint + /// is the only built-in that serves `/v1/responses`. + #[test] + fn responses_api_capability_is_coupled_to_the_preset_list() { + for provider in BUILTIN_CLOUD_PROVIDERS { + let expected = provider.slug == "openai"; + assert_eq!( + builtin_cloud_supports_responses_api(provider.slug), + expected, + "built-in {} Responses-API capability drifted from the openai-only invariant", + provider.slug + ); + } + } + + #[test] + fn endpoint_host_parses_scheme_path_and_port() { + assert_eq!( + endpoint_host("https://integrate.api.nvidia.com/v1").as_deref(), + Some("integrate.api.nvidia.com") + ); + // Missing scheme, mixed case, trailing path. + assert_eq!( + endpoint_host("API.OpenAI.com/v1/chat").as_deref(), + Some("api.openai.com") + ); + // Userinfo + explicit port are stripped. + assert_eq!( + endpoint_host("https://user:pass@api.groq.com:443/openai/v1").as_deref(), + Some("api.groq.com") + ); + // Bracketed IPv6 literal with port. + assert_eq!( + endpoint_host("http://[::1]:8080/v1").as_deref(), + Some("::1") + ); + assert_eq!(endpoint_host(" ").as_deref(), None); + } + + /// TAURI-RUST-HW1: the backend-URL resolver uses this to reroute backend + /// domain calls away from a BYO inference host. Every built-in provider host + /// must be recognised; OpenHuman backend hosts and unknown proxies must not. + #[test] + fn host_is_builtin_cloud_provider_recognises_inference_hosts() { + for host in [ + "openrouter.ai", + "api.openai.com", + "api.anthropic.com", + "api.groq.com", + "generativelanguage.googleapis.com", + "API.OPENAI.COM", // case-insensitive + ] { + assert!( + host_is_builtin_cloud_provider(host), + "{host} is a built-in cloud inference host" + ); + } + for host in [ + "api.tinyhumans.ai", + "staging-api.tinyhumans.ai", + "my-backend.example", + "", + ] { + assert!( + !host_is_builtin_cloud_provider(host), + "{host:?} is not a built-in cloud inference host" + ); + } + // Every registry endpoint's own host must classify as builtin. + for provider in BUILTIN_CLOUD_PROVIDERS { + let host = endpoint_host(provider.endpoint).expect("preset endpoint has a host"); + assert!( + host_is_builtin_cloud_provider(&host), + "{} ({host}) must be recognised", + provider.slug + ); + } + } + + /// TAURI-RUST-5A1: a *custom* slug pointed at a known chat-only host (NVIDIA) + /// must be classified chat-only so the factory disables the guaranteed-404 + /// `/responses` fallback — the builtin-slug gate alone misses this because + /// the slug is not builtin. + #[test] + fn nvidia_host_is_chat_completions_only_regardless_of_slug() { + assert!(endpoint_host_is_chat_completions_only( + "https://integrate.api.nvidia.com/v1" + )); + // Other chat-only built-in hosts too. + for endpoint in [ + "https://api.deepseek.com/v1", + "https://api.groq.com/openai/v1", + "https://api.mistral.ai/v1", + ] { + assert!( + endpoint_host_is_chat_completions_only(endpoint), + "{endpoint} is a chat-completions-only built-in host" + ); + } + } + + #[test] + fn openai_host_and_unknown_proxies_keep_the_responses_fallback() { + // OpenAI's first-party host serves /responses — must NOT be gated off, + // even via a custom proxy slug pointed at it. + assert!(!endpoint_host_is_chat_completions_only( + "https://api.openai.com/v1" + )); + // Genuinely unknown proxy hosts keep the permissive default (they may be + // real OpenAI proxies that implement /responses). + for endpoint in [ + "https://my-llm-proxy.internal.example/v1", + "https://litellm.mycorp.dev/v1", + "", + ] { + assert!( + !endpoint_host_is_chat_completions_only(endpoint), + "{endpoint:?} is an unknown host and must keep the fallback" + ); + } + } + + /// Drift guard: the host-based gate must agree with the slug-based + /// capability for every built-in preset's own endpoint, so adding a preset + /// can't silently desync the two gates. + #[test] + fn host_gate_agrees_with_slug_capability_for_every_builtin() { + for provider in BUILTIN_CLOUD_PROVIDERS { + // OpenhumanJwt / Anthropic presets never route through the + // OpenAI-compatible Responses fallback; the gate only matters for + // the Bearer OpenAI-compatible hosts. + if provider.auth_style != AuthStyle::Bearer { + continue; + } + let host_chat_only = endpoint_host_is_chat_completions_only(provider.endpoint); + let slug_supports = builtin_cloud_supports_responses_api(provider.slug); + assert_eq!( + host_chat_only, !slug_supports, + "host gate for built-in {} disagrees with its slug capability", + provider.slug + ); + } + } +} diff --git a/src/openhuman/memory/api/host/composio.rs b/src/openhuman/memory/api/host/composio.rs new file mode 100644 index 0000000000..2cbec2a958 --- /dev/null +++ b/src/openhuman/memory/api/host/composio.rs @@ -0,0 +1,878 @@ +//! Composio value types — connections, capabilities, execute responses. +//! +//! Moved here from the host's `integrations::composio::types` because the +//! extracted memory sync pipelines read these fields directly on every run, and +//! a trait accessor per field would be absurd. They are inert serde data with +//! no behaviour and no dependencies beyond `serde`, so the contract crate's +//! dependency-light guarantee is unaffected. +//! +//! The Composio *client* deliberately did not come with them — see +//! `crate::openhuman::memory::core_impl::composio_host`. Its `Direct` variant wraps a host agent +//! tool, and mode dispatch is host policy. +//! +//! Domain types for the Composio integration. +//! +//! These mirror the response envelopes emitted by the openhuman backend under +//! `/agent-integrations/composio/*`. See: +//! - `src/routes/agentIntegrations/composio.ts` +//! - `src/controllers/agentIntegrations/composio/*.ts` +//! in the backend repo for the authoritative shapes. + +use serde::{Deserialize, Deserializer, Serialize}; + +/// Accepts either a JSON string or an object whose first matching field +/// (`slug`/`id`/`name`/`key`) is a string. Lets us tolerate upstream +/// shape drift where a previously-stringy field is now nested in an +/// object — e.g. `"toolkit": {"slug": "gmail", "logo": "…"}`. +fn de_string_or_object<'de, D: Deserializer<'de>>(d: D) -> Result { + use serde::de::Error; + let v = serde_json::Value::deserialize(d)?; + match v { + serde_json::Value::String(s) => Ok(s), + serde_json::Value::Object(map) => { + for key in ["slug", "id", "name", "key"] { + if let Some(serde_json::Value::String(s)) = map.get(key) { + return Ok(s.clone()); + } + } + Err(D::Error::custom( + "expected string or object with slug/id/name/key field", + )) + } + other => Err(D::Error::custom(format!( + "expected string, got {}", + match other { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "bool", + serde_json::Value::Number(_) => "number", + serde_json::Value::Array(_) => "array", + _ => "unknown", + } + ))), + } +} + +/// Like [`de_string_or_object`] but optional and resilient: missing / +/// null / unrecognized object shapes return `None` instead of erroring. +fn de_opt_string_or_object<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let v = Option::::deserialize(d)?; + Ok(match v { + None | Some(serde_json::Value::Null) => None, + Some(serde_json::Value::String(s)) => Some(s), + Some(serde_json::Value::Object(map)) => { + let mut found = None; + for key in ["state", "value", "slug", "id", "name", "key"] { + if let Some(serde_json::Value::String(s)) = map.get(key) { + found = Some(s.clone()); + break; + } + } + found + } + _ => None, + }) +} + +// ── Toolkits ──────────────────────────────────────────────────────── + +/// One toolkit from the live Composio catalog, forwarded verbatim from the +/// backend (`GET /agent-integrations/composio/toolkits`). +/// +/// The core does not interpret these fields — it passes them straight through +/// to the desktop UI so the app no longer hardcodes toolkit display metadata +/// (see the workspace `COMPOSIO_DYNAMIC_CATALOG_PLAN.md`). Everything except +/// `slug` is best-effort; backends predating the dynamic catalog omit the +/// whole `catalog` array, in which case the UI falls back to local metadata. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioToolkitCatalogEntry { + /// Toolkit slug as Composio emits it, e.g. `"googlecalendar"`. + pub slug: String, + /// Human-readable name, e.g. `"Google Calendar"`. + #[serde(default)] + pub name: String, + /// Composio-hosted logo URL (`meta.logo`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logo: Option, + /// Short description (`meta.description`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Composio category names (`meta.categories`). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub categories: Vec, + /// Whether the user can connect/use this toolkit (passed the backend gate). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub enabled: Option, +} + +/// Response body of `GET /agent-integrations/composio/toolkits`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioToolkitsResponse { + /// Server-enforced toolkit allowlist, e.g. `["gmail", "notion"]`. + #[serde(default)] + pub toolkits: Vec, + /// Rich render model from the live Composio catalog. Optional — empty when + /// the backend predates the dynamic catalog. Forwarded as-is to the UI. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub catalog: Vec, +} + +/// One row in OpenHuman's local Composio capability matrix. +/// +/// Unlike `ComposioToolkitsResponse`, this is not tied to a signed-in +/// backend/direct Composio session. It describes what this core build knows +/// how to do for each toolkit: whether the toolkit has a native provider +/// implementation, a curated tool catalog, profile/sync hooks, and memory +/// ingestion support. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioCapability { + pub toolkit: String, + pub description: String, + pub native_provider: bool, + pub curated_tools: bool, + pub curated_tool_count: usize, + pub tool_execution: bool, + pub user_profile: bool, + pub initial_sync: bool, + pub periodic_sync: bool, + pub sync_interval_secs: Option, + pub trigger_webhooks: bool, + pub memory_ingest: bool, +} + +/// Response body of `composio.list_capabilities`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioCapabilitiesResponse { + #[serde(default)] + pub capabilities: Vec, +} + +/// Response body of `composio.list_agent_ready_toolkits`. +/// +/// Sorted slugs that have a curated agent catalog — the frontend +/// uses this to decide whether to label a connected toolkit as +/// "preview / agent integration coming soon". See #2283. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioAgentReadyToolkitsResponse { + #[serde(default)] + pub toolkits: Vec, +} + +// ── Connections ───────────────────────────────────────────────────── + +/// One connected Composio account (OAuth integration instance). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioConnection { + /// Composio connection id (what you DELETE to disconnect). + pub id: String, + /// Toolkit slug, e.g. `"gmail"`. + pub toolkit: String, + /// Connection status — `"ACTIVE"`, `"CONNECTED"`, `"PENDING"`, … + pub status: String, + /// ISO timestamp (backend passes this through from Composio). + #[serde(rename = "createdAt", default, skip_serializing_if = "Option::is_none")] + pub created_at: Option, + /// Account email — populated from the cached provider profile when + /// the toolkit reports an email address (e.g. Gmail, Google Calendar, + /// Google Sheets). Lets the UI picker show "Gmail · user@example.com" + /// instead of a generic "Account N" label. + #[serde( + rename = "accountEmail", + default, + skip_serializing_if = "Option::is_none" + )] + pub account_email: Option, + /// Workspace or team display name — populated for workspace-based + /// services (e.g. Slack: user display name / team name, Notion: workspace + /// name). Used by the picker when no email is available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, + /// Screen name or handle — populated for username-based services + /// (e.g. GitHub login, Twitter handle). Used by the picker as a + /// last-resort identity hint after email and workspace. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub username: Option, +} + +impl ComposioConnection { + /// Return the toolkit slug in the canonical form used by provider + /// lookup, prompt injection, and tool-action prefix matching. + pub fn normalized_toolkit(&self) -> String { + self.toolkit.trim().to_ascii_lowercase() + } + + /// Whether this row represents a usable connection. + /// + /// The web UI already treats status case-insensitively. Keep the + /// core-side chat/runtime filters aligned so a backend spelling such + /// as `connected` cannot display as connected in Settings while + /// disappearing from the agent's integration surface. + pub fn is_active(&self) -> bool { + let status = self.status.trim(); + status.eq_ignore_ascii_case("ACTIVE") || status.eq_ignore_ascii_case("CONNECTED") + } +} + +/// Response body of `GET /agent-integrations/composio/connections`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioConnectionsResponse { + #[serde(default)] + pub connections: Vec, +} + +/// Response body of `POST /agent-integrations/composio/authorize`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioAuthorizeResponse { + /// Composio-hosted OAuth URL the user opens in a browser. + #[serde(rename = "connectUrl")] + pub connect_url: String, + /// Composio connection id created by this authorize call. + #[serde(rename = "connectionId")] + pub connection_id: String, +} + +/// Response body of `DELETE /agent-integrations/composio/connections/:id`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioDeleteResponse { + #[serde(default)] + pub deleted: bool, + #[serde(default)] + pub memory_chunks_deleted: usize, +} + +// ── Tools ─────────────────────────────────────────────────────────── + +/// OpenAI function-calling schema returned by the backend for each tool. +/// +/// The backend wraps Composio's upstream shape; we keep the `type` + +/// `function` envelope so callers can forward directly into an LLM. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioToolSchema { + #[serde(rename = "type", default = "default_function_type")] + pub kind: String, + pub function: ComposioToolFunction, +} + +fn default_function_type() -> String { + "function".to_string() +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioToolFunction { + /// Composio action slug, e.g. `"GMAIL_SEND_EMAIL"`. + pub name: String, + /// Human-readable description shown to the model. + #[serde(default)] + pub description: Option, + /// JSON schema for the tool's INPUT parameters. + #[serde(default)] + pub parameters: Option, + /// JSON schema describing the tool's OUTPUT/return-value shape, when the + /// upstream listing publishes one. Composio's v3 `/tools` endpoint calls + /// this `output_parameters` — documented as "Schema definition of return + /// values from the tool" + /// () — + /// alongside `input_parameters`. `None` means "unknown" (not "empty"): + /// the backend-proxied `/agent-integrations/composio/tools` path is + /// opaque to this crate and may not forward it, and not every Composio + /// action publishes an output schema. + #[serde(default)] + pub output_parameters: Option, +} + +/// Response body of `GET /agent-integrations/composio/tools`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioToolsResponse { + #[serde(default)] + pub tools: Vec, +} + +// ── Execute ───────────────────────────────────────────────────────── + +/// Response body of `POST /agent-integrations/composio/execute`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioExecuteResponse { + /// Raw result from the upstream provider. + #[serde(default)] + pub data: serde_json::Value, + /// Did the provider report success? + #[serde(default)] + pub successful: bool, + /// Provider error message if any. + #[serde(default)] + pub error: Option, + /// Amount charged to the caller (base + margin) in USD. + #[serde(rename = "costUsd", default)] + pub cost_usd: f64, + /// Backend-rendered compact markdown for known tools (set by + /// backend PR tinyhumansai/backend#683). When present and non-empty + /// callers should prefer this over `data` for LLM/CLI consumption. + #[serde(rename = "markdownFormatted", default)] + pub markdown_formatted: Option, +} + +// ── GitHub repos + triggers ───────────────────────────────────────── + +/// One repository returned by `GET /agent-integrations/composio/github/repos`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioGithubRepo { + pub owner: String, + pub repo: String, + #[serde(rename = "fullName")] + pub full_name: String, + #[serde(default)] + pub private: Option, + #[serde(rename = "defaultBranch", default)] + pub default_branch: Option, + #[serde(rename = "htmlUrl", default)] + pub html_url: Option, +} + +/// Response body of `GET /agent-integrations/composio/github/repos`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioGithubReposResponse { + #[serde(rename = "connectionId")] + pub connection_id: String, + #[serde(default, rename = "repositories")] + pub repositories: Vec, +} + +/// Response body of `POST /agent-integrations/composio/triggers`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioCreateTriggerResponse { + #[serde(rename = "triggerId")] + pub trigger_id: String, + #[serde(default)] + pub status: Option, +} + +// ── Trigger management (catalog + active list + enable/disable) ───── + +/// Per-repo descriptor used by GitHub-scoped available triggers. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioAvailableTriggerRepo { + pub owner: String, + pub repo: String, +} + +/// One entry in `GET /agent-integrations/composio/triggers/available`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioAvailableTrigger { + pub slug: String, + /// `"static"` or `"github_repo"`. + pub scope: String, + #[serde( + rename = "defaultConfig", + default, + skip_serializing_if = "Option::is_none" + )] + pub default_config: Option, + #[serde( + rename = "requiredConfigKeys", + default, + skip_serializing_if = "Option::is_none" + )] + pub required_config_keys: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioAvailableTriggersResponse { + #[serde(default)] + pub triggers: Vec, +} + +/// One entry in `GET /agent-integrations/composio/triggers`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioActiveTrigger { + #[serde(deserialize_with = "de_string_or_object")] + pub id: String, + #[serde(deserialize_with = "de_string_or_object")] + pub slug: String, + #[serde(deserialize_with = "de_string_or_object")] + pub toolkit: String, + #[serde(rename = "connectionId", deserialize_with = "de_string_or_object")] + pub connection_id: String, + #[serde( + rename = "triggerConfig", + default, + skip_serializing_if = "Option::is_none" + )] + pub trigger_config: Option, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "de_opt_string_or_object" + )] + pub state: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioActiveTriggersResponse { + #[serde(default)] + pub triggers: Vec, +} + +/// Response body of `POST /agent-integrations/composio/triggers` (enable). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioEnableTriggerResponse { + #[serde(rename = "triggerId")] + pub trigger_id: String, + pub slug: String, + #[serde(rename = "connectionId")] + pub connection_id: String, +} + +/// Response body of `DELETE /agent-integrations/composio/triggers/:id`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioDisableTriggerResponse { + #[serde(default)] + pub deleted: bool, +} + +// ── Triggers ──────────────────────────────────────────────────────── + +/// Payload of the `composio:trigger` Socket.IO event emitted by the backend +/// when a Composio webhook is received, HMAC-verified, and delivered to the +/// user's active sockets. +/// +/// See `src/controllers/agentIntegrations/composio/handleWebhook.ts` in the +/// backend repo. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioTriggerEvent { + /// Toolkit slug, e.g. `"gmail"`. + #[serde(default)] + pub toolkit: String, + /// Trigger slug, e.g. `"GMAIL_NEW_GMAIL_MESSAGE"`. + #[serde(default)] + pub trigger: String, + /// Trigger-specific payload (provider-defined shape). + #[serde(default)] + pub payload: serde_json::Value, + /// Metadata the backend attaches: `{ id, uuid }`. + #[serde(default)] + pub metadata: ComposioTriggerMetadata, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioTriggerMetadata { + #[serde(default)] + pub id: String, + #[serde(default)] + pub uuid: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioTriggerHistoryEntry { + /// Unix timestamp in milliseconds when the trigger reached the core. + pub received_at_ms: u64, + /// Toolkit slug, e.g. `"gmail"`. + pub toolkit: String, + /// Trigger slug, e.g. `"GMAIL_NEW_GMAIL_MESSAGE"`. + pub trigger: String, + /// Backend metadata id for this event. + pub metadata_id: String, + /// Backend metadata UUID for this event. + pub metadata_uuid: String, + /// Raw provider payload as forwarded by the backend socket event. + pub payload: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioTriggerHistoryResult { + /// Directory containing daily JSONL archives. + pub archive_dir: String, + /// Today's JSONL file path. + pub current_day_file: String, + /// Recent triggers, newest first. + pub entries: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn connection_is_active_matches_ui_status_normalization() { + for status in ["ACTIVE", "CONNECTED", "active", "connected", " connected "] { + let conn = ComposioConnection { + id: "c1".into(), + toolkit: "slack".into(), + status: status.into(), + created_at: None, + account_email: None, + workspace: None, + username: None, + }; + assert!(conn.is_active(), "status {status:?} should be active"); + } + + for status in ["PENDING", "INITIATED", "FAILED", ""] { + let conn = ComposioConnection { + id: "c1".into(), + toolkit: "slack".into(), + status: status.into(), + created_at: None, + account_email: None, + workspace: None, + username: None, + }; + assert!(!conn.is_active(), "status {status:?} should not be active"); + } + } + + #[test] + fn connection_normalizes_toolkit_for_runtime_matching() { + let conn = ComposioConnection { + id: "c1".into(), + toolkit: " Slack ".into(), + status: "ACTIVE".into(), + created_at: None, + account_email: None, + workspace: None, + username: None, + }; + assert_eq!(conn.normalized_toolkit(), "slack"); + } + + #[test] + fn toolkits_response_defaults_to_empty() { + let resp: ComposioToolkitsResponse = serde_json::from_str("{}").unwrap(); + assert!(resp.toolkits.is_empty()); + } + + #[test] + fn toolkits_response_roundtrips() { + let resp = ComposioToolkitsResponse { + toolkits: vec!["gmail".into(), "notion".into()], + ..Default::default() + }; + let value = serde_json::to_value(&resp).unwrap(); + // Empty catalog is skipped on the wire — back-compat with old cores. + assert_eq!(value, json!({ "toolkits": ["gmail", "notion"] })); + let back: ComposioToolkitsResponse = serde_json::from_value(value).unwrap(); + assert_eq!(back.toolkits, vec!["gmail", "notion"]); + assert!(back.catalog.is_empty()); + } + + #[test] + fn toolkits_response_forwards_catalog() { + // A backend that sends the dynamic catalog must deserialize and + // re-serialize verbatim so the field reaches the desktop UI. + let raw = json!({ + "toolkits": ["gmail"], + "catalog": [ + { + "slug": "gmail", + "name": "Gmail", + "logo": "https://logos.composio.dev/api/gmail", + "description": "Send and read email", + "categories": ["productivity"], + "enabled": true + } + ] + }); + let resp: ComposioToolkitsResponse = serde_json::from_value(raw).unwrap(); + assert_eq!(resp.catalog.len(), 1); + let entry = &resp.catalog[0]; + assert_eq!(entry.slug, "gmail"); + assert_eq!(entry.name, "Gmail"); + assert_eq!(entry.enabled, Some(true)); + assert_eq!(entry.categories, vec!["productivity".to_string()]); + + // Round-trips back out with the catalog intact. + let value = serde_json::to_value(&resp).unwrap(); + assert_eq!(value["catalog"][0]["slug"], "gmail"); + assert_eq!(value["catalog"][0]["enabled"], true); + } + + #[test] + fn connection_parses_and_serializes_camelcase_created_at() { + let raw = json!({ + "id": "conn_1", + "toolkit": "gmail", + "status": "ACTIVE", + "createdAt": "2026-02-01T00:00:00Z" + }); + let conn: ComposioConnection = serde_json::from_value(raw.clone()).unwrap(); + assert_eq!(conn.id, "conn_1"); + assert_eq!(conn.toolkit, "gmail"); + assert_eq!(conn.status, "ACTIVE"); + assert_eq!(conn.created_at.as_deref(), Some("2026-02-01T00:00:00Z")); + + // Round-trip must use camelCase too. + let serialized = serde_json::to_value(&conn).unwrap(); + assert!(serialized.get("createdAt").is_some()); + } + + #[test] + fn connection_without_created_at_omits_field_when_serialized() { + let conn = ComposioConnection { + id: "x".into(), + toolkit: "notion".into(), + status: "PENDING".into(), + created_at: None, + account_email: None, + workspace: None, + username: None, + }; + let s = serde_json::to_value(&conn).unwrap(); + assert!( + s.get("createdAt").is_none(), + "createdAt must be skipped when None" + ); + } + + #[test] + fn authorize_response_uses_camelcase_keys() { + let raw = json!({ + "connectUrl": "https://composio.dev/oauth/abc", + "connectionId": "conn_2" + }); + let resp: ComposioAuthorizeResponse = serde_json::from_value(raw).unwrap(); + assert_eq!(resp.connect_url, "https://composio.dev/oauth/abc"); + assert_eq!(resp.connection_id, "conn_2"); + + let s = serde_json::to_value(&resp).unwrap(); + assert!(s.get("connectUrl").is_some()); + assert!(s.get("connectionId").is_some()); + } + + #[test] + fn tool_schema_defaults_type_field_to_function() { + let raw = json!({ + "function": { + "name": "GMAIL_SEND_EMAIL", + "description": "Send an email", + "parameters": { "type": "object" } + } + }); + let tool: ComposioToolSchema = serde_json::from_value(raw).unwrap(); + assert_eq!(tool.kind, "function"); + assert_eq!(tool.function.name, "GMAIL_SEND_EMAIL"); + assert_eq!(tool.function.description.as_deref(), Some("Send an email")); + assert!(tool.function.parameters.is_some()); + } + + #[test] + fn tool_function_tolerates_missing_description_and_parameters() { + let raw = json!({ "function": { "name": "SLUG_ONLY" } }); + let tool: ComposioToolSchema = serde_json::from_value(raw).unwrap(); + assert_eq!(tool.function.name, "SLUG_ONLY"); + assert!(tool.function.description.is_none()); + assert!(tool.function.parameters.is_none()); + } + + #[test] + fn execute_response_parses_cost_and_error() { + let raw = json!({ + "data": { "messageId": "m-1" }, + "successful": true, + "error": null, + "costUsd": 0.0025 + }); + let resp: ComposioExecuteResponse = serde_json::from_value(raw).unwrap(); + assert!(resp.successful); + assert!(resp.error.is_none()); + assert!((resp.cost_usd - 0.0025).abs() < f64::EPSILON); + } + + #[test] + fn execute_response_defaults_when_fields_missing() { + let resp: ComposioExecuteResponse = serde_json::from_str("{}").unwrap(); + assert!(!resp.successful); + assert!(resp.error.is_none()); + assert_eq!(resp.cost_usd, 0.0); + assert!(resp.data.is_null()); + } + + #[test] + fn available_trigger_deserializes_and_serializes_camelcase_fields() { + let raw = json!({ + "slug": "GMAIL_NEW_GMAIL_MESSAGE", + "scope": "static", + "defaultConfig": { "labelIds": ["INBOX"] }, + "requiredConfigKeys": ["labelIds"], + "repo": { "owner": "acme", "repo": "inbox" } + }); + let trigger: ComposioAvailableTrigger = serde_json::from_value(raw).unwrap(); + assert_eq!(trigger.slug, "GMAIL_NEW_GMAIL_MESSAGE"); + assert_eq!(trigger.scope, "static"); + assert_eq!( + trigger.default_config, + Some(json!({ "labelIds": ["INBOX"] })) + ); + assert_eq!( + trigger.required_config_keys, + Some(vec!["labelIds".to_string()]) + ); + let repo = trigger.repo.as_ref().expect("repo"); + assert_eq!(repo.owner, "acme"); + assert_eq!(repo.repo, "inbox"); + + let value = serde_json::to_value(&trigger).unwrap(); + assert!(value.get("defaultConfig").is_some()); + assert!(value.get("requiredConfigKeys").is_some()); + } + + #[test] + fn active_trigger_parses_connection_id_and_optional_fields() { + let raw = json!({ + "id": "ti_1", + "slug": "GMAIL_NEW_GMAIL_MESSAGE", + "toolkit": "gmail", + "connectionId": "c-1", + "triggerConfig": { "labelIds": "INBOX" }, + "state": "active" + }); + let trigger: ComposioActiveTrigger = serde_json::from_value(raw).unwrap(); + assert_eq!(trigger.id, "ti_1"); + assert_eq!(trigger.slug, "GMAIL_NEW_GMAIL_MESSAGE"); + assert_eq!(trigger.connection_id, "c-1"); + assert_eq!(trigger.trigger_config, Some(json!({"labelIds":"INBOX"}))); + assert_eq!(trigger.state.as_deref(), Some("active")); + + let value = serde_json::to_value(&trigger).unwrap(); + assert!(value.get("connectionId").is_some()); + assert!(value.get("triggerConfig").is_some()); + assert!(value.get("state").is_some()); + } + + #[test] + fn trigger_enable_response_uses_camelcase_and_optional_defaults() { + let raw = json!({ + "triggerId": "ti_9", + "slug": "GMAIL_NEW_GMAIL_MESSAGE", + "connectionId": "c-9" + }); + let resp: ComposioEnableTriggerResponse = serde_json::from_value(raw).unwrap(); + assert_eq!(resp.trigger_id, "ti_9"); + assert_eq!(resp.slug, "GMAIL_NEW_GMAIL_MESSAGE"); + assert_eq!(resp.connection_id, "c-9"); + + let serialized = serde_json::to_value(&resp).unwrap(); + assert_eq!(serialized.get("triggerId").unwrap(), "ti_9"); + assert_eq!(serialized.get("connectionId").unwrap(), "c-9"); + } + + #[test] + fn delete_trigger_response_defaults_deleted_to_false() { + let raw = json!({}); + let resp: ComposioDisableTriggerResponse = serde_json::from_value(raw).unwrap(); + assert!(!resp.deleted); + } + + #[test] + fn trigger_event_defaults_empty_fields_to_empty_strings() { + let ev: ComposioTriggerEvent = serde_json::from_str("{}").unwrap(); + assert_eq!(ev.toolkit, ""); + assert_eq!(ev.trigger, ""); + assert_eq!(ev.metadata.id, ""); + assert_eq!(ev.metadata.uuid, ""); + assert!(ev.payload.is_null()); + } + + #[test] + fn trigger_event_parses_full_payload() { + let raw = json!({ + "toolkit": "gmail", + "trigger": "GMAIL_NEW_GMAIL_MESSAGE", + "payload": { "subject": "hi" }, + "metadata": { "id": "evt-1", "uuid": "uuid-1" } + }); + let ev: ComposioTriggerEvent = serde_json::from_value(raw).unwrap(); + assert_eq!(ev.toolkit, "gmail"); + assert_eq!(ev.trigger, "GMAIL_NEW_GMAIL_MESSAGE"); + assert_eq!(ev.metadata.id, "evt-1"); + assert_eq!(ev.metadata.uuid, "uuid-1"); + assert_eq!(ev.payload["subject"], "hi"); + } + + #[test] + fn active_trigger_accepts_string_fields() { + let v = json!({ + "id": "t1", + "slug": "GMAIL_NEW_MAIL", + "toolkit": "gmail", + "connectionId": "c1", + "state": "ACTIVE", + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert_eq!(trig.id, "t1"); + assert_eq!(trig.slug, "GMAIL_NEW_MAIL"); + assert_eq!(trig.toolkit, "gmail"); + assert_eq!(trig.connection_id, "c1"); + assert_eq!(trig.state.as_deref(), Some("ACTIVE")); + } + + #[test] + fn active_trigger_accepts_object_fields() { + // Mirrors upstream API drift where these fields arrive as objects + // rather than plain strings. + let v = json!({ + "id": {"id": "t1"}, + "slug": {"slug": "GMAIL_NEW_MAIL"}, + "toolkit": {"slug": "gmail", "logo": "https://…"}, + "connectionId": {"id": "c1"}, + "state": {"state": "ACTIVE", "slug": "should-be-ignored"}, + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert_eq!(trig.id, "t1"); + assert_eq!(trig.slug, "GMAIL_NEW_MAIL"); + assert_eq!(trig.toolkit, "gmail"); + assert_eq!(trig.connection_id, "c1"); + // `state` priority must prefer the literal `state` key over metadata. + assert_eq!(trig.state.as_deref(), Some("ACTIVE")); + } + + #[test] + fn active_trigger_state_falls_back_to_value() { + let v = json!({ + "id": "t1", + "slug": "X", + "toolkit": "gmail", + "connectionId": "c1", + "state": {"value": "PENDING"}, + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert_eq!(trig.state.as_deref(), Some("PENDING")); + } + + #[test] + fn active_trigger_state_missing_or_unknown_returns_none() { + let v = json!({ + "id": "t1", + "slug": "X", + "toolkit": "gmail", + "connectionId": "c1", + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert!(trig.state.is_none()); + + let v = json!({ + "id": "t1", + "slug": "X", + "toolkit": "gmail", + "connectionId": "c1", + "state": {"unrelated": 42}, + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert!(trig.state.is_none()); + } + + #[test] + fn active_trigger_required_field_rejects_unsupported_object() { + // Object without any of slug/id/name/key must fail loudly so we + // notice further upstream shape drift instead of silently dropping + // the trigger. + let v = json!({ + "id": {"unrelated": 42}, + "slug": "X", + "toolkit": "gmail", + "connectionId": "c1", + }); + let err = serde_json::from_value::(v).unwrap_err(); + assert!(err.to_string().contains("expected string or object")); + } +} diff --git a/src/openhuman/memory/api/host/config.rs b/src/openhuman/memory/api/host/config.rs new file mode 100644 index 0000000000..7ee21b8104 --- /dev/null +++ b/src/openhuman/memory/api/host/config.rs @@ -0,0 +1,256 @@ +//! [`MemoryHostConfig`] — the memory subsystem's view of the host's config. +//! +//! # Why a trait and not a struct +//! +//! The host's `Config` is one giant serde struct covering voice, channels, +//! sandboxing, inference routing, the agent harness — the lot. The memory +//! subsystem reads about two dozen of its fields. Moving the whole struct into +//! this crate would drag the host's entire configuration vocabulary into a +//! contract crate that is meant to stay dependency-light; leaving it behind and +//! passing individual values would mean rewriting every function signature in +//! the extracted code. +//! +//! A trait threads the needle. `crate::openhuman::memory::core_impl::Config` is the alias +//! `dyn MemoryHostConfig`, so a function that took `config: &Config` before the +//! extraction still takes `config: &Config` after it, and the host's concrete +//! `Config` unsize-coerces at the call site with no edit at all. Only the field +//! *accesses* inside the extracted code change, from `config.workspace_dir` to +//! `config.workspace_dir()`. +//! +//! # Accessor shapes are chosen for zero churn, not for elegance +//! +//! Several accessors return `&PathBuf` / `&Vec` where `&Path` / `&[T]` would +//! be the idiomatic choice. That is deliberate: the extracted code calls +//! `.clone()` on these values in dozens of places, and `&Path`/`&[T]` would +//! silently resolve `.clone()` to the *reference*'s `Clone` impl and fail at the +//! use site with a confusing type error. Returning the owning type keeps every +//! one of those sites compiling unchanged. +//! +//! # Mutation +//! +//! Three methods take `&mut self`. They exist because the extracted code owns +//! two write paths the host does not: the composio source-caps migration and +//! the CLI's env-override re-application. Everything else is read-only. + +use std::path::PathBuf; + +use super::cloud_providers::CloudProviderCreds; +use super::local_ai::LocalAiConfig; +use super::scheduler_gate::SchedulerGateConfig; +use super::storage_memory::{MemoryConfig, MemoryTreeConfig}; + +/// Composio routing mode: proxied through the host's cloud backend. +pub const COMPOSIO_MODE_BACKEND: &str = "backend"; +/// Composio routing mode: BYO API key, calling `backend.composio.dev` directly. +pub const COMPOSIO_MODE_DIRECT: &str = "direct"; + +/// The subset of a host's Composio configuration the memory sync pipelines read. +/// +/// Passed by value rather than by reference because the host's own +/// `ComposioConfig` carries fields (toolkit triage opt-outs, the enabled flag) +/// that have nothing to do with memory, and because borrowing it would pin the +/// host's type into this contract. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ComposioMode { + /// [`COMPOSIO_MODE_BACKEND`] or [`COMPOSIO_MODE_DIRECT`]. + pub mode: String, + /// The Composio entity the host authenticates as. + pub entity_id: String, + /// Direct-mode API key, when the user hand-wrote one into `config.toml`. + /// The keychain-backed value takes precedence and is resolved host-side. + pub api_key: Option, + /// Whether the LLM triage turn is switched off for all triggers. + pub triage_disabled: bool, +} + +impl ComposioMode { + /// True when the host routes Composio calls directly rather than through + /// its cloud backend. + #[must_use] + pub fn is_direct(&self) -> bool { + self.mode.eq_ignore_ascii_case(COMPOSIO_MODE_DIRECT) + } +} + +/// The host's configuration, as the memory subsystem sees it. +/// +/// Implemented by the embedding application for its own root config type. See +/// the module docs for why this is a trait and why the accessor return types +/// are shaped the way they are. +#[async_trait::async_trait] +pub trait MemoryHostConfig: Send + Sync + std::fmt::Debug { + // ── Paths ─────────────────────────────────────────────────────────────── + + /// Root of the host's internal per-user state. Every memory database, + /// summary-tree directory and queue file is resolved beneath this. + fn workspace_dir(&self) -> &PathBuf; + + /// Absolute path of the `config.toml` this config was loaded from. + fn config_path(&self) -> &PathBuf; + + /// Where chunk `.md` files are written. Either the explicit + /// `memory_tree.content_dir` or `/memory_tree/content`. + fn memory_tree_content_root(&self) -> PathBuf; + + // ── Memory-owned sections ─────────────────────────────────────────────── + + /// The `[memory]` block — backend selection, embedding provider/model/dims, + /// relevance floor, SQLite timeouts. + fn memory(&self) -> &MemoryConfig; + + /// The `[memory_tree]` block — summary-tree embedder, extractor and + /// summariser wiring. + fn memory_tree(&self) -> &MemoryTreeConfig; + + /// The `[scheduler_gate]` block — when background LLM-bound work may run. + fn scheduler_gate(&self) -> &SchedulerGateConfig; + + // ── Host-owned sections the memory subsystem still reads ──────────────── + // + // These are the seam's rough edge (see the module docs on `host`): they are + // read only to *construct* embedding providers, which is work that belongs + // in the host. Moving the embedding factory back out of the core would let + // all four of these accessors go away. + + /// The `[local_ai]` block — whether a local runtime is enabled and which + /// model it serves. + fn local_ai(&self) -> &LocalAiConfig; + + /// Configured cloud LLM/embedding backends, keyed by user-chosen slug. + fn cloud_providers(&self) -> &Vec; + + /// `provider:model` routing string for the embeddings workload, if pinned. + fn embeddings_provider(&self) -> Option<&str>; + + /// `provider:model` routing string for the memory workload, if pinned. + fn memory_provider(&self) -> Option<&str>; + + /// The local model id for a workload, when that workload is routed to + /// Ollama (`"ollama:"`). `None` for cloud or unset workloads. + /// + /// This is the single source of truth for "is this workload local?" — + /// callers must not consult the deprecated `local_ai.usage.*` booleans or + /// `memory_tree.llm_backend`. + fn workload_local_model(&self, workload: &str) -> Option; + + // ── Scalars ───────────────────────────────────────────────────────────── + + /// The concrete config behind this trait object, for host code that needs + /// its own type back. + /// + /// The seam deliberately hands the core a `dyn MemoryHostConfig`, and that + /// is the right shape for everything the core does. But a *host* + /// implementation of one of the behavioural seams — chat-model routing, + /// Composio mode dispatch — is handed the same trait object and has to get + /// its own `Config` back: routing reads BYOK fallbacks, per-role routes and + /// credentials, none of which are on this trait and none of which should + /// be. + /// + /// Implementations return `self`. A host downcasts and, on failure, falls + /// back to whatever it was configured with — a failure means the config is + /// somebody else's type (a test double), not that something is wrong. + fn as_any(&self) -> &dyn std::any::Any; + + /// An owned, shareable handle to this config. + /// + /// `crate::openhuman::memory::core_impl::Config` is the *unsized* `dyn MemoryHostConfig`, which + /// makes `&Config` free at every call site — the host's concrete `Config` + /// unsize-coerces with no edit. The cost is that a borrow cannot be turned + /// into an owned value: background loops that outlive their caller, structs + /// that hold a config, and `spawn_blocking` bodies all need one. + /// + /// This is that escape hatch. Implementations return + /// `Arc::new(self.clone())`; callers that only read should keep taking + /// `&Config` rather than reaching for this. + fn to_arc(&self) -> std::sync::Arc; + + /// Backend base URL, used to recognise first-party endpoints. + fn api_url(&self) -> Option<&str>; + + /// The backend API URL this host actually talks to, with the host's own + /// environment and default resolution already applied. + /// + /// Distinct from [`Self::api_url`], which is the raw configured value — + /// resolution (env override, staging/prod default, trailing-slash + /// normalisation) is host logic and must not be re-derived here. + fn effective_backend_api_url(&self) -> String; + + /// The current backend session bearer, or `None` when signed out. + /// + /// Read through the trait rather than from a config field because the host + /// keeps it in its credential store, not in `config.toml`. + /// + /// # Errors + /// + /// Returns `Err` when the credential store cannot be read — distinct from + /// `Ok(None)`, which means "read fine, not signed in". + fn session_token(&self) -> Result, String>; + + /// Default chat model id. + fn default_model(&self) -> Option<&str>; + + /// Default sampling temperature for background LLM calls. + fn default_temperature(&self) -> f64; + + /// Optional language for background LLM artifacts — tree summaries, + /// extraction reasons, learning reflections. `None` keeps the default. + fn output_language(&self) -> Option<&str>; + + /// Global memory-sync cadence in seconds. `None` means "no explicit choice" + /// and callers fall back to [`super::DEFAULT_MEMORY_SYNC_INTERVAL_SECS`]; + /// `Some(0)` means manual-only. + fn memory_sync_interval_secs(&self) -> Option; + + /// Whether the user has finished onboarding. Background ingestion holds off + /// until they have. + fn onboarding_completed(&self) -> bool; + + /// Whether at-rest secret encryption is switched on for this workspace. + fn secrets_encrypt(&self) -> bool; + + /// Composio routing mode + credentials, as the sync pipelines need them. + fn composio(&self) -> ComposioMode; + + // ── Memory sources ────────────────────────────────────────────────────── + // + // Serde-mediated on purpose. `MemorySourceEntry` is defined by the *engine* + // crate (`tinycortex`), which this contract crate must not depend on — it + // would drag SQLite in and break the dependency-light guarantee this crate + // exists to hold. JSON is the narrowest waist that keeps the type where it + // belongs. + + /// The persisted `[[memory_sources]]` registry, as JSON. + /// + /// # Errors + /// Propagates a serialization failure from the host's own entry type. + fn memory_sources_json(&self) -> anyhow::Result; + + /// Replace the persisted `[[memory_sources]]` registry. Does not save to + /// disk — call [`Self::save`] afterwards. + /// + /// # Errors + /// Returns an error when `value` does not deserialize into the host's entry + /// type, in which case the registry is left untouched. + fn set_memory_sources_json(&mut self, value: serde_json::Value) -> anyhow::Result<()>; + + // ── Migration bookkeeping ─────────────────────────────────────────────── + + /// Version of the composio source-capabilities migration already applied. + fn composio_source_caps_migration_version(&self) -> u32; + + /// Record that the composio source-capabilities migration has run. + fn set_composio_source_caps_migration_version(&mut self, version: u32); + + // ── Lifecycle ─────────────────────────────────────────────────────────── + + /// Re-apply the host's environment-variable overlay over this config. + /// Used by the CLI entry points, which build a config before the host's + /// normal load path has run. + fn apply_env_overrides(&mut self); + + /// Persist this config back to [`Self::config_path`] atomically. + /// + /// # Errors + /// Propagates the host's own write/serialize failure. + async fn save(&self) -> anyhow::Result<()>; +} diff --git a/src/openhuman/memory/api/host/embedding_host.rs b/src/openhuman/memory/api/host/embedding_host.rs new file mode 100644 index 0000000000..55451a762c --- /dev/null +++ b/src/openhuman/memory/api/host/embedding_host.rs @@ -0,0 +1,101 @@ +//! [`EmbeddingHost`] — provider *construction*, which the host owns. +//! +//! [`super::EmbeddingProvider`] is the contract for a provider that already +//! exists. This trait is the other half: how one comes into being. Resolving an +//! API key from the credential store, knowing which managed cloud endpoint the +//! signed-in user is entitled to, knowing where the local Ollama server is +//! listening — all of that is host policy, and none of it belongs in a memory +//! engine. +//! +//! The core reaches this through a process-global installed at startup, for the +//! same reason [`super::MemoryEventSink`] is a global: the construction sites +//! sit deep inside retrieval and sealing call stacks that already thread a +//! config and a store handle. +//! +//! # Default is failure, not silence +//! +//! Unlike the event sink, an unwired [`EmbeddingHost`] must **not** degrade +//! quietly. A missing sink drops a notification about work that already +//! happened; a missing embedding provider means vectors would be written into +//! the wrong embedding space, or a query would silently return lexical-only +//! results. Both are data corruption with a delayed fuse, so the unwired +//! accessors return `Err`/`None` and every call site is written to propagate. + +use std::sync::Arc; + +use super::EmbeddingProvider; + +/// Builds [`EmbeddingProvider`]s on the core's behalf. +/// +/// Object-safe: the core holds one as `Arc`. +pub trait EmbeddingHost: Send + Sync + std::fmt::Debug { + /// The API key for `provider`, from the host's credential store. + /// + /// Returns `None` when the provider has no stored credential — which is not + /// an error: a local provider needs none, and an unconfigured cloud one is + /// a state the caller reports rather than a failure. + fn resolve_api_key(&self, provider: &str) -> Option; + + /// Base URL of the local Ollama server, honouring the host's env override + /// and config before falling back to the default. + fn ollama_base_url(&self) -> String; + + /// The host's default provider — the managed cloud embedder. + /// + /// Constructed lazily with respect to authentication: this may be called + /// before login completes, and the first `embed()` is what fails if the + /// user is unauthenticated. + fn default_embedding_provider(&self) -> Arc; + + /// Builds a provider from an explicit provider/model/credential triple. + /// + /// # Errors + /// + /// Returns `Err` when `provider` is not one the host knows how to build, or + /// when the supplied credentials are unusable for it. + fn create_embedding_provider_with_credentials( + &self, + provider: &str, + model: &str, + dims: usize, + api_key: &str, + custom_endpoint: Option<&str>, + ) -> Result, String>; + + /// Whether `model` accepts a caller-chosen output dimensionality. + /// + /// Asking for dimensions a model does not support is rejected by the + /// provider at request time, so the core checks first rather than writing a + /// batch that will fail halfway. + fn model_supports_dimensions(&self, model: &str) -> bool; + + /// The managed cloud embedder at an explicit model and dimensionality. + /// + /// # Errors + /// + /// Returns `Err` when the host cannot reach its managed endpoint + /// configuration. + fn cloud_embedding_provider( + &self, + model: &str, + dims: usize, + ) -> Result, String>; + + /// The default model id the managed cloud embedder uses. + fn default_cloud_embedding_model(&self) -> &str; + + /// The dimensionality [`Self::default_cloud_embedding_model`] emits. + fn default_cloud_embedding_dimensions(&self) -> usize; + + /// An Ollama-backed provider at `base_url`. + /// + /// # Errors + /// + /// Returns `Err` when the host cannot construct one for `model`. + fn ollama_embedding_provider( + &self, + base_url: &str, + model: &str, + dims: usize, + ) -> Result, String>; +} diff --git a/src/openhuman/memory/api/host/embeddings.rs b/src/openhuman/memory/api/host/embeddings.rs new file mode 100644 index 0000000000..d5cc55bcf4 --- /dev/null +++ b/src/openhuman/memory/api/host/embeddings.rs @@ -0,0 +1,110 @@ +//! [`EmbeddingProvider`] — text → vector, supplied by the host. +//! +//! The memory subsystem embeds chunks, summaries and queries, but it does not +//! decide *how*: which provider, which credentials, which rate limit and which +//! fallback are host policy. So the core takes an `Arc` +//! and never constructs one. +//! +//! This trait deliberately lives in the contract crate rather than in +//! `tinymemory-core`, so that a host implementing it does not have to depend on +//! the engine. It carries nothing heavier than `async-trait` and `anyhow`. + +use async_trait::async_trait; + +/// Formats the canonical embedding-space signature string. +/// +/// This is the **single source of truth** for the signature format. Both the +/// live-provider [`EmbeddingProvider::signature`] and any config-derived +/// signature must route through here, so a signature computed from +/// configuration is byte-identical to one computed from an instantiated +/// provider. Drift between the two silently splits one embedding space into +/// two, and every vector written on the wrong side of the split becomes +/// unsearchable without a re-embed. +#[must_use] +pub fn format_embedding_signature(name: &str, model_id: &str, dims: usize) -> String { + format!( + "provider={}:{};model={}:{};dims={dims}", + name.len(), + name, + model_id.len(), + model_id + ) +} + +#[cfg(test)] +mod tests { + use super::format_embedding_signature; + + #[test] + fn delimiter_characters_cannot_make_distinct_spaces_collide() { + let first = format_embedding_signature("a;model=b", "c", 3); + let second = format_embedding_signature("a", "b;model=c", 3); + assert_ne!(first, second); + } +} + +/// Converts text into numerical vectors. +#[async_trait] +pub trait EmbeddingProvider: Send + Sync { + /// Provider name, e.g. `"ollama"`, `"openai"`. + fn name(&self) -> &str; + + /// Stable model identifier used to generate embeddings. + fn model_id(&self) -> &str; + + /// Number of dimensions in the generated embeddings. + fn dimensions(&self) -> usize; + + /// Stable signature for the embedding space. + /// + /// Changing any component means existing vectors are no longer comparable + /// with newly generated ones and must be stored and queried separately + /// until a migration re-embeds them. + fn signature(&self) -> String { + format_embedding_signature(self.name(), self.model_id(), self.dimensions()) + } + + /// Generates embeddings for a batch of strings. + /// + /// # Errors + /// Propagates transport, authentication and quota failures from the + /// underlying provider. + async fn embed(&self, texts: &[&str]) -> anyhow::Result>>; + + /// Generates an embedding for a single string. + /// + /// # Errors + /// As [`Self::embed`], plus an error when the provider returns no vector. + async fn embed_one(&self, text: &str) -> anyhow::Result> { + let mut results = self.embed(&[text]).await?; + results + .pop() + .ok_or_else(|| anyhow::anyhow!("Empty embedding result")) + } +} + +/// The inert provider bound when semantic search is switched off or no +/// embedding backend is configured. Reports zero dimensions and returns one +/// empty vector per input, so keyword-only retrieval keeps working while +/// vector rerank degrades to a no-op rather than an error. +#[derive(Debug, Clone, Copy, Default)] +pub struct NoopEmbedding; + +#[async_trait] +impl EmbeddingProvider for NoopEmbedding { + fn name(&self) -> &str { + "none" + } + + fn model_id(&self) -> &str { + "none" + } + + fn dimensions(&self) -> usize { + 0 + } + + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(vec![Vec::new(); texts.len()]) + } +} diff --git a/src/openhuman/memory/api/host/error_reporter.rs b/src/openhuman/memory/api/host/error_reporter.rs new file mode 100644 index 0000000000..02c874ba8e --- /dev/null +++ b/src/openhuman/memory/api/host/error_reporter.rs @@ -0,0 +1,43 @@ +//! [`ErrorReporter`] — the host's crash/error telemetry, as the core sees it. +//! +//! The memory subsystem reports a handful of failures that are worth a +//! developer's attention: a corrupt SQLite database, host filesystem I/O +//! errors, a sync run that failed for a non-user reason. *Where* those go — +//! Sentry, a log sink, nowhere — and which of them count as expected rather +//! than exceptional is host policy, so the core states the fact and the host +//! decides what to do with it. +//! +//! # The two methods are not interchangeable +//! +//! [`ErrorReporter::report_error`] is unconditional: the caller has already +//! decided this is a real defect. [`ErrorReporter::report_error_or_expected`] +//! asks the host to classify first, so routine user- and config-caused failures +//! (an unreachable local runtime, a revoked OAuth token) do not page anyone. +//! Collapsing them into one would either spam the error channel or hide real +//! bugs, which is why both exist. + +/// Receives error reports from the memory subsystem. +/// +/// Takes the **already-rendered** message rather than a concrete error type: +/// the trait has to be object-safe, so it cannot be generic over `E: Display` +/// the way the host's own `report_error` is. The core's free functions keep +/// that generic signature and render with `{:#}` — the alternate specifier that +/// makes `anyhow::Error` print its full context chain — before crossing. +pub trait ErrorReporter: Send + Sync + std::fmt::Debug { + /// Report `error` as a defect worth investigating. + /// + /// `domain` and `operation` are stable, low-cardinality strings used for + /// grouping (`"memory"` / `"tree_jobs_worker_corrupt"`); `tags` carries + /// additional non-sensitive key/value context. + fn report_error(&self, rendered: &str, domain: &str, operation: &str, tags: &[(&str, &str)]); + + /// Report `error`, letting the host classify it as a defect or an expected + /// user/config failure and route it accordingly. + fn report_error_or_expected( + &self, + rendered: &str, + domain: &str, + operation: &str, + tags: &[(&str, &str)], + ); +} diff --git a/src/openhuman/memory/api/host/events.rs b/src/openhuman/memory/api/host/events.rs new file mode 100644 index 0000000000..a843db43bd --- /dev/null +++ b/src/openhuman/memory/api/host/events.rs @@ -0,0 +1,228 @@ +//! [`MemoryEventSink`] — the events the memory subsystem announces. +//! +//! # Why the host's event enum does not move +//! +//! The host's `DomainEvent` is a single flat enum covering agents, channels, +//! cron, tools, webhooks and the system domain as well as memory. It is the +//! host's own vocabulary; a *memory* crate must not own it, and importing it +//! would make every other subsystem's events a transitive dependency of memory. +//! +//! So the seam runs the other way. This module defines the ~15 memory-domain +//! events the extracted code emits, as a small enum of plain data. The host +//! implements [`MemoryEventSink`] by mapping each variant onto the matching +//! `DomainEvent` and publishing it on its own bus. The core publishes into the +//! sink and never learns that a bus exists. +//! +//! # Subscribing is not part of this seam +//! +//! Several extracted modules used to *subscribe* as well as publish +//! (`sync_events.rs`, `sync/composio/bus.rs`, `conversations/bus.rs`). Those are +//! host wiring by the repository README's split — event-bus subscribers belong +//! in the host, next to the registration site that installs them. They move back +//! rather than growing a subscribe method here. + +/// Why an embedding model was reported unhealthy, and what took over. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct EmbeddingHealthReason { + /// The provider that failed. + pub provider: String, + /// The model that failed. + pub model: String, + /// The provider bound in its place. + pub fallback_provider: String, + /// Operator-facing explanation. Never carries credentials. + pub message: String, +} + +/// What kicked off a sync run — a schedule, a user action, a webhook. +pub type SyncTrigger = String; + +/// A memory-domain event, as announced by `tinymemory-core`. +/// +/// Field names and types mirror the host's own event payloads exactly, so the +/// host's [`MemoryEventSink`] impl is a straight structural mapping with no +/// judgement calls in it. +/// +/// Deliberately **not** `#[non_exhaustive]`: the host's mapping impl matches +/// exhaustively on purpose, so adding a variant here is a compile error at the +/// mapping site rather than an event that silently never reaches the bus. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum MemoryEvent { + /// A sync run moved to a new stage. + SyncStageChanged { + /// What started this run. + trigger: SyncTrigger, + /// The stage just entered. + stage: String, + /// Provider slug, when the stage is provider-scoped. + provider: Option, + /// Connection id, when the stage is connection-scoped. + connection_id: Option, + /// Free-form operator detail. + detail: Option, + /// Memory-source id, when the stage is source-scoped. + source_id: Option, + }, + /// A document entered the ingestion pipeline. + IngestionStarted { + /// Document being ingested. + document_id: String, + /// Human-readable document title. + title: String, + /// Target namespace. + namespace: String, + /// Items still queued behind this one. + queue_depth: usize, + }, + /// A document left the ingestion pipeline. + IngestionCompleted { + /// Document that was ingested. + document_id: String, + /// Target namespace. + namespace: String, + /// Whether ingestion succeeded. + success: bool, + /// Wall-clock duration. + elapsed_ms: u64, + /// Items still queued afterwards. + queue_depth: usize, + }, + /// A source document was canonicalized into chunks. + DocumentCanonicalized { + /// Source the document came from. + source_id: String, + /// Source kind (`gmail`, `slack`, `file`, …). + source_kind: String, + /// How many chunks were written. + chunks_written: usize, + /// Ids of the written chunks. + chunk_ids: Vec, + /// Unix timestamp, seconds with fraction. + canonicalized_at: f64, + /// Truncated body preview for operator UIs. + body_preview: Option, + }, + /// An hour bucket was sealed and summarized. + TreeSummarizerHourCompleted { + /// Tree namespace. + namespace: String, + /// Node that was sealed. + node_id: String, + /// Tokens in the produced summary. + token_count: u32, + }, + /// A summary was propagated up a level. + TreeSummarizerPropagated { + /// Tree namespace. + namespace: String, + /// Node that received the propagated summary. + node_id: String, + /// Level name. + level: String, + /// Tokens in the produced summary. + token_count: u32, + }, + /// A full tree rebuild finished. + TreeSummarizerRebuildCompleted { + /// Tree namespace. + namespace: String, + /// Nodes in the rebuilt tree. + total_nodes: u64, + }, + /// Progress ticks during a tree build, for the operator UI. + TreeBuildProgress { + /// Coarse phase name. + phase: String, + /// Fine step name. + step: String, + /// Which tree, when scoped. + tree_scope: Option, + /// Tree level, when levelled. + level: Option, + /// Items processed in this step. + item_count: Option, + /// Free-form operator detail. + detail: Option, + }, + /// An embedding model failed health checks and a fallback was bound. + EmbeddingModelUnhealthy(EmbeddingHealthReason), + /// The configured memory driver could not be bound, and another was used. + DriverBindFailed { + /// Driver named in config. + configured_driver: String, + /// Driver actually bound. + bound_driver: String, + /// Why the configured driver was rejected. + reason: String, + }, + /// A diff snapshot was captured for a source. + DiffSnapshotTaken { + /// The new snapshot. + snapshot_id: String, + /// Source the snapshot covers. + source_id: String, + /// Source kind. + source_kind: String, + /// Items in the snapshot. + item_count: usize, + /// What triggered the snapshot. + trigger: String, + }, + /// Diffs were acknowledged by the user. + DiffMarkedRead { + /// Sources marked read. + source_ids: Vec, + /// Snapshots marked read. + snapshot_ids: Vec, + }, + /// The set of connected Composio toolkits changed. + ComposioIntegrationsChanged { + /// Toolkit slugs now connected. + toolkits: Vec, + }, + /// The memory subsystem is asking for a sync run. + SyncRequested { + /// Channel to report progress back on, when the request came from one. + channel_id: Option, + }, + /// The local embedding runtime is unusable and the user must act outside + /// the app (start Ollama, pull the model). + /// + /// The host surfaces this in its durable user-error centre. Carries no + /// provider text, model id or endpoint — see [`LOCAL_MODEL_UNAVAILABLE_KIND`]. + LocalModelUnavailable { + /// Short, non-sensitive tag naming which producer fired + /// (`health_gate` / `embed_classify`), so the two paths stay + /// distinguishable in the log without a correlation id. + origin: String, + }, +} + +/// Stable `error_type` token for the local-embedding-runtime user error. +/// +/// Mirrors the frontend `UserErrorKind` discriminator of the same name. It is +/// defined in the contract crate because both sides name it: the host builds +/// the wire payload from it, and the core's tests assert on it. A drift on +/// either side drops the signal silently. +pub const LOCAL_MODEL_UNAVAILABLE_KIND: &str = "local_model_unavailable"; + +/// `error_source` for the memory subsystem's user errors. Drives the panel's +/// scope grouping (`socketService` maps it to the `memory` `UserErrorScope`). +pub const MEMORY_USER_ERROR_SOURCE: &str = "memory"; + +/// Receives [`MemoryEvent`]s and does something host-shaped with them. +pub trait MemoryEventSink: Send + Sync + std::fmt::Debug { + /// Announce an event. Implementations must not block and must not fail — + /// an event bus that can reject a publish turns every emit site into an + /// error path, which is not what any of the call sites want. + fn publish(&self, event: MemoryEvent); +} + +/// The sink bound when no host has installed one — in unit tests, in the +/// standalone engine build, and before startup wiring runs. Drops everything. +#[derive(Debug, Clone, Copy, Default)] +pub struct NoopEventSink; + +impl MemoryEventSink for NoopEventSink { + fn publish(&self, _event: MemoryEvent) {} +} diff --git a/src/openhuman/memory/api/host/evidence.rs b/src/openhuman/memory/api/host/evidence.rs new file mode 100644 index 0000000000..7134b6d6a0 --- /dev/null +++ b/src/openhuman/memory/api/host/evidence.rs @@ -0,0 +1,49 @@ +//! [`EvidenceRef`] — a pointer to the thing a learned fact was learned from. +//! +//! Moved here from the host's `agent::learning::candidate` because it is +//! persisted *in the memory store*: `store::namespace_store::profile` writes it +//! into profile rows, and the Composio provider-profile sync reads it back. Two +//! structurally identical enums either side of the seam would round-trip +//! through serde and silently diverge on the first added variant. +//! +//! Inert serde data; the contract crate's dependency-light guarantee is +//! unaffected. **Its serde form is persisted**, so the `#[serde(tag = "type")]` +//! representation and every variant name are a compatibility surface. + +use serde::{Deserialize, Serialize}; + +/// A typed pointer back into the memory substrate from which a candidate was +/// derived. Used for provenance tracking, citation, and the `evidence_ids` +/// column in `user_profile_facets` (Phase 3+). +/// +/// Serialised with a `"type"` discriminator in snake_case so the JSON is +/// human-readable: `{"type":"episodic","episodic_id":42}`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EvidenceRef { + /// A single row in `episodic_log`. + Episodic { episodic_id: i64 }, + /// A contiguous window of rows in `episodic_log`. + EpisodicWindow { from_id: i64, to_id: i64 }, + /// A row in the tree-source summary table. + SourceSummary { summary_id: String }, + /// A node in `tree_topic`. + TreeTopic { topic_id: String }, + /// A chunk in `vector_chunks` associated with a document source. + DocumentChunk { source_id: String, chunk_id: String }, + /// A specific message in an email source. + EmailMessage { + source_id: String, + message_id: String, + }, + /// A field value from a connected provider (Composio toolkit). + Provider { + toolkit: String, + connection_id: String, + field: String, + }, + /// A tool call record within an episodic entry. + ToolCall { tool_name: String, episodic_id: i64 }, + /// A per-window weight from `tree_source`. + TreeSourceWeight { window_label: String }, +} diff --git a/src/openhuman/memory/api/host/local_ai.rs b/src/openhuman/memory/api/host/local_ai.rs new file mode 100644 index 0000000000..f5cee1e7f6 --- /dev/null +++ b/src/openhuman/memory/api/host/local_ai.rs @@ -0,0 +1,285 @@ +//! Local AI runtime configuration. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Per-feature flags controlling which subsystems route through the selected +/// local runtime. All default to `false` (use cloud instead). Guarded by +/// `LocalAiConfig::runtime_enabled` — when that is `false` every helper +/// method below returns `false` regardless of these values. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +#[derive(Default)] +pub struct LocalAiUsage { + /// When true (and `runtime_enabled`), use the local model for embedding + /// generation instead of the cloud backend. + #[serde(default)] + pub embeddings: bool, + /// When true (and `runtime_enabled`), use the local model inside the + /// heartbeat loop. + #[serde(default)] + pub heartbeat: bool, + /// When true (and `runtime_enabled`), use the local model for + /// learning/reflection passes. + #[serde(default)] + pub learning_reflection: bool, + /// When true (and `runtime_enabled`), use the local model for + /// subconscious evaluation and execution. + #[serde(default)] + pub subconscious: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct LocalAiConfig { + /// Master runtime switch. Defaults to `false` — local AI is OFF by default. + /// Note: the old on-disk field was `enabled`; that key is now unknown to + /// serde and will be silently ignored on load (intentional forced reset). + #[serde(default = "default_runtime_enabled")] + pub runtime_enabled: bool, + /// Local provider identifier. Supported values are `ollama`, `lm_studio`, + /// and `omlx`; unknown values normalize to `ollama` at runtime. + #[serde(default = "default_provider")] + pub provider: String, + /// Optional provider base URL. For LM Studio this defaults to + /// `http://localhost:1234/v1`. + #[serde(default)] + pub base_url: Option, + #[serde(default)] + pub api_key: Option, + #[serde(default = "default_model_id")] + pub model_id: String, + #[serde(default = "default_chat_model_id")] + pub chat_model_id: String, + #[serde(default = "default_vision_model_id")] + pub vision_model_id: String, + #[serde(default = "default_embedding_model_id")] + pub embedding_model_id: String, + #[serde(default = "default_stt_model_id")] + pub stt_model_id: String, + #[serde(default = "default_stt_download_url")] + pub stt_download_url: Option, + /// Legacy voice STT routing string. `"cloud"` (the default) means "use + /// `voice_server.stt_engine`"; a third-party `"[:]"` overrides + /// the engine outright. The local `"whisper"` value it once accepted is + /// dead — `config::migrations` rewrites it back to `"cloud"`. + #[serde(default = "default_stt_provider")] + pub stt_provider: String, + #[serde(default = "default_tts_voice_id")] + pub tts_voice_id: String, + /// Voice TTS provider selector. `"cloud"` (default) routes through the + /// backend ElevenLabs proxy and returns rich visemes; `"piper"` runs + /// local Piper via the `PIPER_BIN` env var. + #[serde(default = "default_tts_provider")] + pub tts_provider: String, + #[serde(default = "default_tts_download_url")] + pub tts_download_url: Option, + #[serde(default = "default_tts_config_download_url")] + pub tts_config_download_url: Option, + #[serde(default = "default_quantization")] + pub quantization: String, + #[serde(default = "default_preload_vision_model")] + pub preload_vision_model: bool, + #[serde(default = "default_preload_embedding_model")] + pub preload_embedding_model: bool, + #[serde(default = "default_preload_stt_model")] + pub preload_stt_model: bool, + #[serde(default = "default_preload_tts_voice")] + pub preload_tts_voice: bool, + #[serde(default = "default_download_url")] + pub download_url: Option, + #[serde(default = "default_autosummary_debounce_ms")] + pub autosummary_debounce_ms: u64, + #[serde(default)] + pub selected_tier: Option, + /// Explicit MVP opt-in marker. Bootstrap disables local AI unless this is + /// `true`, regardless of any prior `selected_tier` value. Existing installs + /// (upgrading from pre-MVP) default to `false` and must re-opt-in from + /// Settings. Set by `apply_preset` on any non-disabled tier. + #[serde(default)] + pub opt_in_confirmed: bool, + /// Optional path to a manually-installed Ollama binary. + #[serde(default)] + pub ollama_binary_path: Option, + /// When true and Ollama is available, pass raw transcription through a + /// local LLM to fix grammar/punctuation using conversation context. + #[serde(default = "default_voice_llm_cleanup_enabled")] + pub voice_llm_cleanup_enabled: bool, + /// Ollama `options.num_ctx` override. When set, every chat request to + /// an Ollama provider includes `"options": {"num_ctx": }` so + /// the model allocates at least this much KV-cache. Ollama defaults + /// to 2048 for many models which is too small for agentic use. + #[serde(default)] + pub num_ctx: Option, + /// Per-feature flags. Each gate is AND-ed with `runtime_enabled`. + /// All default to `false` (cloud path). + #[serde(default)] + pub usage: LocalAiUsage, +} + +fn default_runtime_enabled() -> bool { + false +} + +fn default_provider() -> String { + "ollama".to_string() +} + +fn default_model_id() -> String { + "gemma3:1b-it-qat".to_string() +} + +fn default_chat_model_id() -> String { + "gemma3:1b-it-qat".to_string() +} + +fn default_vision_model_id() -> String { + String::new() +} + +fn default_embedding_model_id() -> String { + // bge-m3 (1024 dims, 8192-token context). Required by the memory tree's + // fixed on-disk embedding format (EMBEDDING_DIM=1024) — `all-minilm` + // (384 dims) and `nomic-embed-text` (768 dims) would fail the + // post-call dim validator at `memory::tree::score::embed::mod::embed`. + "bge-m3".to_string() +} + +fn default_stt_model_id() -> String { + "ggml-base-q5_1.bin".to_string() +} + +fn default_tts_voice_id() -> String { + "en_US-lessac-medium".to_string() +} + +fn default_stt_provider() -> String { + "cloud".to_string() +} + +fn default_tts_provider() -> String { + "cloud".to_string() +} + +fn default_stt_download_url() -> Option { + Some( + "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q5_1.bin?download=true" + .to_string(), + ) +} + +fn default_tts_download_url() -> Option { + Some( + "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx?download=true" + .to_string(), + ) +} + +fn default_tts_config_download_url() -> Option { + Some( + "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json?download=true" + .to_string(), + ) +} + +fn default_quantization() -> String { + "q4".to_string() +} + +fn default_preload_vision_model() -> bool { + false +} + +fn default_preload_embedding_model() -> bool { + true +} + +fn default_preload_stt_model() -> bool { + false +} + +fn default_preload_tts_voice() -> bool { + false +} + +fn default_download_url() -> Option { + None +} + +fn default_autosummary_debounce_ms() -> u64 { + 2500 +} + +fn default_voice_llm_cleanup_enabled() -> bool { + true +} + +impl LocalAiConfig { + /// Returns `true` when the local Ollama runtime is active. + /// This is the primary gate; all per-feature helpers below AND with this. + pub fn is_active(&self) -> bool { + self.runtime_enabled + } + + /// **Deprecated** — read from `Config::workload_uses_local("embeddings")` + /// instead. This helper only consults the legacy `usage.*` booleans, which + /// are no longer the source of truth after the unified AI settings + /// migration (schema_version >= 2). + #[deprecated(note = "Use Config::workload_uses_local(\"embeddings\")")] + pub fn use_local_for_embeddings(&self) -> bool { + self.runtime_enabled && self.usage.embeddings + } + + /// **Deprecated** — read from `Config::workload_uses_local("heartbeat")`. + #[deprecated(note = "Use Config::workload_uses_local(\"heartbeat\")")] + pub fn use_local_for_heartbeat(&self) -> bool { + self.runtime_enabled && self.usage.heartbeat + } + + /// **Deprecated** — read from `Config::workload_uses_local("learning")`. + #[deprecated(note = "Use Config::workload_uses_local(\"learning\")")] + pub fn use_local_for_learning(&self) -> bool { + self.runtime_enabled && self.usage.learning_reflection + } + + /// **Deprecated** — read from `Config::workload_uses_local("subconscious")`. + #[deprecated(note = "Use Config::workload_uses_local(\"subconscious\")")] + pub fn use_local_for_subconscious(&self) -> bool { + self.runtime_enabled && self.usage.subconscious + } +} + +impl Default for LocalAiConfig { + fn default() -> Self { + Self { + runtime_enabled: default_runtime_enabled(), + provider: default_provider(), + base_url: None, + api_key: None, + model_id: default_model_id(), + chat_model_id: default_chat_model_id(), + vision_model_id: default_vision_model_id(), + embedding_model_id: default_embedding_model_id(), + stt_model_id: default_stt_model_id(), + stt_download_url: default_stt_download_url(), + stt_provider: default_stt_provider(), + tts_voice_id: default_tts_voice_id(), + tts_provider: default_tts_provider(), + tts_download_url: default_tts_download_url(), + tts_config_download_url: default_tts_config_download_url(), + quantization: default_quantization(), + preload_vision_model: default_preload_vision_model(), + preload_embedding_model: default_preload_embedding_model(), + preload_stt_model: default_preload_stt_model(), + preload_tts_voice: default_preload_tts_voice(), + download_url: default_download_url(), + autosummary_debounce_ms: default_autosummary_debounce_ms(), + selected_tier: None, + opt_in_confirmed: false, + ollama_binary_path: None, + voice_llm_cleanup_enabled: default_voice_llm_cleanup_enabled(), + num_ctx: None, + usage: LocalAiUsage::default(), + } + } +} diff --git a/src/openhuman/memory/api/host/mod.rs b/src/openhuman/memory/api/host/mod.rs new file mode 100644 index 0000000000..5d3e1d6f9b --- /dev/null +++ b/src/openhuman/memory/api/host/mod.rs @@ -0,0 +1,93 @@ +//! The **host seam** — everything `tinymemory-core` needs from the application +//! that embeds it, expressed as object-safe traits plus the plain serde config +//! structs the memory subsystem owns. +//! +//! # Why this module exists +//! +//! `tinymemory-core` holds the substance of a memory subsystem: the store, the +//! summary tree, the sync pipelines, ingestion, recall. Per the repository +//! README's split, the *host* keeps the RPC surface, the agent tools, the +//! security policy, the schedulers, the event bus, and config loading. That +//! split only works if the core can name what it needs from the host without +//! naming the host itself — which is what these traits are. +//! +//! # The three seams +//! +//! - [`MemoryHostConfig`] — the host's configuration, read through accessor +//! methods rather than public fields. `crate::openhuman::memory::core_impl::Config` is the type +//! alias `dyn MemoryHostConfig`, so code moved out of the host keeps writing +//! `config: &Config` and the host's concrete `Config` unsize-coerces at every +//! call site. +//! - [`EmbeddingProvider`] — text → vector. The core never builds one; the host +//! resolves provider credentials, rate limits and routing and hands an +//! `Arc` down. +//! - [`MemoryEventSink`] — the handful of domain events the memory subsystem +//! publishes. The host implements it by publishing its own event enum onto +//! its own bus; the core never learns that enum exists. +//! +//! # Config *sections* live here, config *loading* does not +//! +//! [`MemoryConfig`], [`MemoryTreeConfig`], [`MemorySubsystemConfig`] and friends +//! moved here from the host because the core reads their fields directly and a +//! trait accessor per field would be absurd. They are inert serde/`schemars` +//! data with no behaviour, and **their serde representation is persisted in +//! users' `config.toml`** — field names, defaults, and `#[serde(...)]` +//! attributes are a compatibility surface, not an implementation detail. +//! +//! Sections that are *not* memory-owned but that the core still reads +//! ([`LocalAiConfig`], [`cloud_providers`]) are here for the same mechanical +//! reason. They are the seam's rough edge: the honest fix is to move embedding +//! *construction* back into the host, at which point the core stops reading +//! them and they can go home. + +pub mod cloud_providers; +pub mod composio; +pub mod local_ai; +pub mod scheduler_gate; +pub mod storage_memory; +pub mod subsystems; + +mod config; +mod embedding_host; +mod embeddings; +mod error_reporter; +mod events; +mod evidence; +mod nlp; +mod routes; +mod usage; + +#[cfg(test)] +pub mod test_support; + +pub use cloud_providers::{ + endpoint_host, generate_provider_id, is_slug_reserved, migrate_legacy_fields, AuthStyle, + CloudProviderCreds, CloudProviderType, +}; +pub use config::{ComposioMode, MemoryHostConfig, COMPOSIO_MODE_BACKEND, COMPOSIO_MODE_DIRECT}; +pub use embedding_host::EmbeddingHost; +pub use embeddings::{format_embedding_signature, EmbeddingProvider, NoopEmbedding}; +pub use error_reporter::ErrorReporter; +pub use events::{ + EmbeddingHealthReason, MemoryEvent, MemoryEventSink, NoopEventSink, SyncTrigger, + LOCAL_MODEL_UNAVAILABLE_KIND, MEMORY_USER_ERROR_SOURCE, +}; +pub use evidence::EvidenceRef; +pub use local_ai::{LocalAiConfig, LocalAiUsage}; +pub use nlp::{SpacyEntity, SpacyResponse}; +pub use routes::EmbeddingRouteConfig; +pub use scheduler_gate::{PauseReason, Policy, SchedulerGateConfig, SchedulerGateMode}; +pub use storage_memory::{ + LlmBackend, MemoryConfig, MemoryTreeConfig, StorageConfig, StorageProviderConfig, + StorageProviderSection, DEFAULT_CLOUD_LLM_MODEL, +}; +pub use subsystems::{ + MemoryDriverConfig, MemoryHooksConfig, MemorySubsystemConfig, SubsystemsConfig, +}; +pub use usage::UsageInfo; + +/// Effective default global memory-sync cadence (seconds) used when +/// [`MemoryHostConfig::memory_sync_interval_secs`] is `None` — i.e. the user has +/// not explicitly picked a schedule. 24h, matching the "Sync every 24h" preset +/// surfaced in the Memory Sources UI. +pub const DEFAULT_MEMORY_SYNC_INTERVAL_SECS: u64 = 86_400; diff --git a/src/openhuman/memory/api/host/nlp.rs b/src/openhuman/memory/api/host/nlp.rs new file mode 100644 index 0000000000..46e66605c4 --- /dev/null +++ b/src/openhuman/memory/api/host/nlp.rs @@ -0,0 +1,30 @@ +//! spaCy extraction results — the wire shape of the host's Python NLP server. +//! +//! Moved here from the host's `runtime::python_server::spacy` because the +//! summary tree's query-entity extractor consumes them directly, canonicalising +//! each entity into the same `:` namespace the indexed chunks use. +//! Inert serde data. +//! +//! Provisioning the runtime (`ensure_spacy`, `spacy_provisioned`, the model id) +//! deliberately stayed in the host: downloading and launching a Python server +//! is not something a memory engine should do. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SpacyEntity { + pub text: String, + pub label: String, + #[serde(default)] + pub start: u32, + #[serde(default)] + pub end: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SpacyResponse { + #[serde(default)] + pub entities: Vec, + #[serde(default)] + pub nouns: Vec, +} diff --git a/src/openhuman/memory/api/host/routes.rs b/src/openhuman/memory/api/host/routes.rs new file mode 100644 index 0000000000..f5204e5736 --- /dev/null +++ b/src/openhuman/memory/api/host/routes.rs @@ -0,0 +1,18 @@ +//! [`EmbeddingRouteConfig`] — a per-workload embedding provider override. +//! +//! Moved here from the host's `config::schema::routes` because the memory +//! store's factory reads its fields directly when resolving which embedder +//! backs a workload. Inert serde data; **its serde form is persisted** in +//! users' `config.toml`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct EmbeddingRouteConfig { + pub hint: String, + pub provider: String, + pub model: String, + #[serde(default)] + pub dimensions: Option, +} diff --git a/src/openhuman/memory/api/host/scheduler_gate.rs b/src/openhuman/memory/api/host/scheduler_gate.rs new file mode 100644 index 0000000000..9d3b4e3c12 --- /dev/null +++ b/src/openhuman/memory/api/host/scheduler_gate.rs @@ -0,0 +1,188 @@ +//! Scheduler-gate configuration — controls when background AI work runs. +//! +//! Consumed by `openhuman::cron::scheduler_gate`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +#[derive(Default)] +pub enum SchedulerGateMode { + /// Decide based on power + CPU + deployment-mode signals. + #[default] + Auto, + /// Always run background AI flat-out (server / power-user setting). + AlwaysOn, + /// Never run background AI. User can still trigger work explicitly. + Off, +} + +impl SchedulerGateMode { + pub fn as_str(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::AlwaysOn => "always_on", + Self::Off => "off", + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct SchedulerGateConfig { + /// Top-level mode — `auto` (default), `always_on`, or `off`. + #[serde(default)] + pub mode: SchedulerGateMode, + + /// Battery charge floor in `auto` mode, 0.0..=1.0. Below this and not on + /// AC, the gate throttles. Default: 0.80. + #[serde(default = "default_battery_floor")] + pub battery_floor: f32, + + /// CPU busy threshold (recent global usage, 0..100). Above this, the gate + /// throttles even when plugged in. Default: 70.0 (i.e. <30% headroom). + #[serde(default = "default_cpu_busy_threshold")] + pub cpu_busy_threshold_pct: f32, + + /// In `Throttled` mode, sleep this many ms before each LLM-bound job to + /// serialise workers and let the host catch up. Default: 30_000 (30s). + #[serde(default = "default_throttled_backoff_ms")] + pub throttled_backoff_ms: u64, + + /// In `Paused` mode, re-check the policy every this many ms so workers + /// resume promptly when the user toggles the gate back on. Default: + /// 60_000 (60s). + #[serde(default = "default_paused_poll_ms")] + pub paused_poll_ms: u64, + + /// Hard CPU ceiling (recent global usage, 0..100). When the host CPU + /// climbs above this in `auto` mode, the gate flips to + /// `Paused { CpuPressure }` rather than just `Throttled` — every + /// background LLM call is held until the host calms down. Distinct + /// from `cpu_busy_threshold_pct`, which only triggers `Throttled`. + /// Default: 95.0. + #[serde(default = "default_cpu_severe_pct")] + pub cpu_severe_pct: f32, + + /// When `true`, `auto` mode only runs background LLM work while the + /// laptop is on AC power. On battery the gate flips to + /// `Paused { OnBattery }` — no background inference at all, + /// regardless of charge level. + /// + /// Default `false` to preserve the prior behavior (battery-floor + /// based throttling). Power-conscious users who never want + /// background inference on battery can flip this on. + #[serde(default)] + pub require_ac_power: bool, +} + +fn default_battery_floor() -> f32 { + 0.80 +} +fn default_cpu_busy_threshold() -> f32 { + 70.0 +} +fn default_throttled_backoff_ms() -> u64 { + 30_000 +} +fn default_paused_poll_ms() -> u64 { + 60_000 +} +fn default_cpu_severe_pct() -> f32 { + 95.0 +} + +impl Default for SchedulerGateConfig { + fn default() -> Self { + Self { + mode: SchedulerGateMode::default(), + battery_floor: default_battery_floor(), + cpu_busy_threshold_pct: default_cpu_busy_threshold(), + throttled_backoff_ms: default_throttled_backoff_ms(), + paused_poll_ms: default_paused_poll_ms(), + cpu_severe_pct: default_cpu_severe_pct(), + require_ac_power: false, + } + } +} + +// ── Gate decision vocabulary ──────────────────────────────────────────────── +// +// `Policy` and `PauseReason` moved here from the host's +// `cron::scheduler_gate::policy` because the extracted sync loops read them on +// every tick to decide whether to back off. They are inert `Copy` enums with no +// dependencies; the *decision function* that produces a `Policy` from sampled +// signals stays in the host, where the signals are. + +/// Why the gate is currently paused. Carried by [`Policy::Paused`] so +/// downstream consumers (UI, logging, observability) can surface a +/// specific user-facing reason instead of a generic "paused" label. +/// +/// New variants will land alongside #1073's full power-aware work +/// (`OnBattery`, `CpuPressure`); `UserDisabled` covers the existing +/// `SchedulerGateMode::Off` path and `Unknown` is the safe fallback for +/// callers that don't have specific context yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PauseReason { + /// User explicitly turned the gate off in config. + UserDisabled, + /// Host on battery and gate's power-aware mode kicked in (#1073). + OnBattery, + /// CPU pressure exceeded the gate threshold (#1073). + CpuPressure, + /// No active app session — background AI work is suspended until the + /// user signs in again. Trumps every other signal: while signed out + /// the host should do *no* LLM-bound work, period. Set by + /// `gate::set_signed_out(true)` from the credentials lifecycle and + /// from 401-detection sites. + SignedOut, + /// Pause reason not yet classified — placeholder while #1073 is in flight. + Unknown, +} + +impl PauseReason { + pub fn as_str(self) -> &'static str { + match self { + Self::UserDisabled => "user_disabled", + Self::OnBattery => "on_battery", + Self::CpuPressure => "cpu_pressure", + Self::SignedOut => "signed_out", + Self::Unknown => "unknown", + } + } +} + +/// Background-AI scheduling tier. See module docs in `mod.rs` for semantics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Policy { + Aggressive, + Normal, + Throttled, + /// Gate paused. The `reason` is rendered to users in the memory-sync + /// status UI (#1136) and recorded in observability. + Paused { + reason: PauseReason, + }, +} + +impl Policy { + pub fn as_str(self) -> &'static str { + match self { + Self::Aggressive => "aggressive", + Self::Normal => "normal", + Self::Throttled => "throttled", + Self::Paused { .. } => "paused", + } + } + + /// `Some(reason)` when paused, `None` otherwise. Convenience for + /// callers that only need the reason and don't want to pattern-match + /// the whole enum (UI badges, log line construction). + pub fn pause_reason(self) -> Option { + match self { + Self::Paused { reason } => Some(reason), + _ => None, + } + } +} diff --git a/src/openhuman/memory/api/host/storage_memory.rs b/src/openhuman/memory/api/host/storage_memory.rs new file mode 100644 index 0000000000..928f1da038 --- /dev/null +++ b/src/openhuman/memory/api/host/storage_memory.rs @@ -0,0 +1,561 @@ +//! Storage provider and memory configuration. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Debug, Clone, Serialize, Deserialize, Default, JsonSchema)] +#[serde(default)] +pub struct StorageConfig { + #[serde(default)] + pub provider: StorageProviderSection, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, JsonSchema)] +#[serde(default)] +pub struct StorageProviderSection { + #[serde(default)] + pub config: StorageProviderConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +#[derive(Default)] +pub struct StorageProviderConfig { + #[serde(default)] + pub provider: String, +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[allow(clippy::struct_excessive_bools)] +#[serde(default)] +pub struct MemoryConfig { + #[serde(default = "default_memory_backend")] + pub backend: String, + #[serde(default = "default_true")] + pub auto_save: bool, + #[serde(default = "default_embedding_provider")] + pub embedding_provider: String, + #[serde(default = "default_embedding_model")] + pub embedding_model: String, + #[serde(default = "default_embedding_dims")] + pub embedding_dimensions: usize, + /// Outbound embedding-request budget for cloud providers, in requests per + /// minute. Cloud backends (OpenHuman/Voyage, OpenAI, remote `custom:` + /// endpoints) cap requests per account; the client throttles to stay under + /// that quota rather than tripping 429s. `0` disables throttling. Loopback + /// endpoints are always exempt. Env override: + /// `OPENHUMAN_MEMORY_EMBED_RATE_LIMIT`. + #[serde(default = "default_embedding_rate_limit_per_min")] + pub embedding_rate_limit_per_min: u32, + #[serde(default = "default_min_relevance_score")] + pub min_relevance_score: f64, + #[serde(default)] + pub sqlite_open_timeout_secs: Option, + + /// Base URL for the `agentmemory` REST server. Honored only when + /// `backend = "agentmemory"`. Defaults to `http://localhost:3111` + /// (the agentmemory loopback default). + #[serde(default)] + pub agentmemory_url: Option, + + /// Optional bearer token sent as `Authorization: Bearer ` + /// to the agentmemory REST server. When unset, the backend speaks + /// to a local agentmemory daemon without authentication. Setting a + /// secret + a non-loopback host enables the v0.9.12 plaintext-bearer + /// guard semantics on the client side: the backend refuses to send + /// the token over plaintext HTTP when the host is not loopback. + #[serde(default)] + pub agentmemory_secret: Option, + + /// Per-request timeout for the agentmemory REST client, in + /// milliseconds. Defaults to 5000 ms. + #[serde(default)] + pub agentmemory_timeout_ms: Option, +} + +fn default_memory_backend() -> String { + "sqlite".into() +} + +fn default_true() -> bool { + true +} + +fn default_embedding_provider() -> String { + // Default to the OpenHuman backend (Voyage-backed `embedding-v1`) so a + // fresh install works without requiring a local Ollama daemon. Users + // who want fully-local embeddings can flip this to "ollama" in + // `config.toml` or enable `local_ai.usage.embeddings = true`, which is + // wired into the memory factory via `LocalAiConfig::use_local_for_embeddings`. + "cloud".into() +} +fn default_embedding_model() -> String { + // Keep this in sync with `embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_MODEL`. + "embedding-v1".into() +} +fn default_embedding_dims() -> usize { + // Keep this in sync with `embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_DIMENSIONS`. + 1024 +} +fn default_embedding_rate_limit_per_min() -> u32 { + // Cloud embedding backends cap requests at ~60/min per account. Keep in + // sync with `embeddings::rate_limit::DEFAULT_EMBEDDING_RATE_LIMIT_PER_MIN`. + 60 +} +fn default_min_relevance_score() -> f64 { + 0.4 +} + +impl Default for MemoryConfig { + fn default() -> Self { + Self { + backend: default_memory_backend(), + auto_save: default_true(), + embedding_provider: default_embedding_provider(), + embedding_model: default_embedding_model(), + embedding_dimensions: default_embedding_dims(), + embedding_rate_limit_per_min: default_embedding_rate_limit_per_min(), + min_relevance_score: default_min_relevance_score(), + sqlite_open_timeout_secs: None, + agentmemory_url: None, + agentmemory_secret: None, + agentmemory_timeout_ms: None, + } + } +} + +// Manual `Debug` implementation that redacts `agentmemory_secret`. Without +// this, any `format!("{cfg:?}")` / `tracing::debug!(?cfg, ...)` / panic +// message capturing a `MemoryConfig` would dump the bearer token in +// plaintext — directly against the repo rule "Never log secrets, raw +// JWTs, API keys, credentials, or full PII in debug logs". +impl std::fmt::Debug for MemoryConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryConfig") + .field("backend", &self.backend) + .field("auto_save", &self.auto_save) + .field("embedding_provider", &self.embedding_provider) + .field("embedding_model", &self.embedding_model) + .field("embedding_dimensions", &self.embedding_dimensions) + .field( + "embedding_rate_limit_per_min", + &self.embedding_rate_limit_per_min, + ) + .field("min_relevance_score", &self.min_relevance_score) + .field("sqlite_open_timeout_secs", &self.sqlite_open_timeout_secs) + .field("agentmemory_url", &self.agentmemory_url) + .field( + "agentmemory_secret", + &self.agentmemory_secret.as_ref().map(|_| ""), + ) + .field("agentmemory_timeout_ms", &self.agentmemory_timeout_ms) + .finish() + } +} + +/// Which inference backend the memory_tree's LLM calls (extractor + +/// summariser) should use. +/// +/// - `Cloud` (default): route through `providers::router` against the +/// OpenHuman backend with the `summarization-v1` model. No local Ollama +/// required. +/// - `Local`: keep using the legacy Ollama-direct path (the +/// `llm_extractor_endpoint` / `llm_summariser_endpoint` config). Useful +/// for offline development and CI smoke tests. +/// +/// Embedder selection is unchanged — `OllamaEmbedder` (bge-m3) stays +/// local-only and isn't governed by this enum. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +#[derive(Default)] +pub enum LlmBackend { + /// Route through the OpenHuman backend (default). + #[default] + Cloud, + /// Use the local Ollama path configured via `llm_extractor_*` / + /// `llm_summariser_*`. + Local, +} + +impl LlmBackend { + /// Stable wire string for env vars / RPCs / logs. + pub fn as_str(self) -> &'static str { + match self { + Self::Cloud => "cloud", + Self::Local => "local", + } + } + + /// Inverse of [`Self::as_str`]; case-insensitive parse. + pub fn parse(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "cloud" => Ok(Self::Cloud), + "local" => Ok(Self::Local), + other => Err(format!("unknown llm (expected cloud|local): {other}")), + } + } +} + +fn default_llm_backend() -> LlmBackend { + LlmBackend::default() +} + +/// Default model identifier to use when `llm_backend = "cloud"`. Routed +/// through the OpenHuman backend; keep in sync with the backend's +/// summariser model registry. +pub const DEFAULT_CLOUD_LLM_MODEL: &str = "summarization-v1"; + +fn default_cloud_llm_model() -> Option { + Some(DEFAULT_CLOUD_LLM_MODEL.to_string()) +} + +/// Phase 4 memory-tree configuration — embedding provider wiring for the +/// hierarchical memory (#710). +/// +/// When `embedding_endpoint` and `embedding_model` are both set, ingest +/// and bucket-seal route every new chunk/summary through the Ollama +/// embedder before writing. When unset, behaviour depends on +/// `embedding_strict`: +/// - `true` (default): ingest/seal bail with a clear config error. +/// - `false`: fall back to the inert zero-vector embedder and warn. +/// +/// Env overrides apply in `openhuman::config::schema::load`: +/// - `OPENHUMAN_MEMORY_EMBED_ENDPOINT` +/// - `OPENHUMAN_MEMORY_EMBED_MODEL` +/// - `OPENHUMAN_MEMORY_EMBED_TIMEOUT_MS` +/// - `OPENHUMAN_MEMORY_EXTRACT_ENDPOINT` +/// - `OPENHUMAN_MEMORY_EXTRACT_MODEL` +/// - `OPENHUMAN_MEMORY_EXTRACT_TIMEOUT_MS` +/// - `OPENHUMAN_MEMORY_SUMMARISE_ENDPOINT` +/// - `OPENHUMAN_MEMORY_SUMMARISE_MODEL` +/// - `OPENHUMAN_MEMORY_SUMMARISE_TIMEOUT_MS` +/// - `OPENHUMAN_MEMORY_TREE_CONTENT_DIR` (Phase MD-content) +/// - `OPENHUMAN_MEMORY_TREE_LLM_BACKEND` (cloud|local) +/// - `OPENHUMAN_MEMORY_TREE_CLOUD_LLM_MODEL` +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemoryTreeConfig { + /// Ollama endpoint for the embedder (e.g. `http://localhost:11434`). + /// `None` disables the Ollama path — see `embedding_strict` for the + /// resulting behaviour. + #[serde(default = "default_memory_tree_embedding_endpoint")] + pub embedding_endpoint: Option, + + /// Embedding model name. Must produce 768-dim vectors (see + /// `memory::tree::score::embed::EMBEDDING_DIM`). `None` disables + /// the Ollama path. + #[serde(default = "default_memory_tree_embedding_model")] + pub embedding_model: Option, + + /// Per-request timeout for the embedder, in milliseconds. + #[serde(default = "default_memory_tree_embedding_timeout_ms")] + pub embedding_timeout_ms: Option, + + /// When true, ingest/seal refuse to run with embeddings disabled. + /// When false, an inert zero-vector embedder is used and retrieval + /// rerank falls back to scope + recency ordering only. + #[serde(default = "default_memory_tree_embedding_strict")] + pub embedding_strict: bool, + + /// Ollama endpoint for the LLM entity extractor + /// (`memory::tree::score::extract::llm::LlmEntityExtractor`). + /// Defaults to `Some("http://localhost:11434")` — the standard + /// Ollama listener — see `default_memory_tree_llm_endpoint`. + /// Soft failures in the LLM path fall back to regex-only for + /// that chunk. + #[serde(default = "default_memory_tree_llm_endpoint")] + pub llm_extractor_endpoint: Option, + + /// Model name for the entity extractor. Defaults to `gemma3:4b` + /// (see `default_memory_tree_llm_model` for the rationale); + /// override to a smaller model on resource-constrained hosts. + #[serde(default = "default_memory_tree_llm_model")] + pub llm_extractor_model: Option, + + /// Per-request timeout for the LLM extractor, in milliseconds. + #[serde(default = "default_memory_tree_llm_extractor_timeout_ms")] + pub llm_extractor_timeout_ms: Option, + + /// Ollama endpoint for the summariser + /// (`memory::tree::tree_source::summariser::llm::LlmSummariser`). + /// Defaults to `Some("http://localhost:11434")` — see + /// `default_memory_tree_llm_endpoint`. Soft failures fall back + /// to `InertSummariser` per seal. + #[serde(default = "default_memory_tree_llm_endpoint")] + pub llm_summariser_endpoint: Option, + + /// Model name for the summariser. Defaults to `gemma3:4b` — + /// larger Gemma tiers (`gemma3:12b-it-qat`, `gemma3:27b`) produce + /// more coherent abstractive summaries at higher latency. See + /// `default_memory_tree_llm_model`. + #[serde(default = "default_memory_tree_llm_model")] + pub llm_summariser_model: Option, + + /// Per-request timeout for the summariser, in milliseconds. Default + /// is higher than the extractor because summarisation uses more + /// tokens and therefore takes longer to generate. + #[serde(default = "default_memory_tree_llm_summariser_timeout_ms")] + pub llm_summariser_timeout_ms: Option, + + /// Phase MD-content: root directory where chunk `.md` files are stored. + /// + /// Resolved at runtime via `MemoryHostConfig::memory_tree_content_root`: + /// - `Some(path)` → use that path verbatim. + /// - `None` → default `/memory_tree/content/`. + /// + /// Env override: `OPENHUMAN_MEMORY_TREE_CONTENT_DIR` (empty string = fall + /// back to default, consistent with other memory_tree env vars). + #[serde(default = "default_memory_tree_content_dir")] + pub content_dir: Option, + + /// Backend selector for the memory_tree's LLM calls (extractor + + /// summariser). Defaults to [`LlmBackend::Cloud`] so a fresh install + /// works without requiring a local Ollama daemon. Set to + /// [`LlmBackend::Local`] (or `OPENHUMAN_MEMORY_TREE_LLM_BACKEND=local`) to + /// keep the legacy Ollama-direct path. + /// + /// The embedder is unaffected by this setting — `OllamaEmbedder` (bge-m3) + /// stays local-only. + #[serde(default = "default_llm_backend")] + pub llm_backend: LlmBackend, + + /// **Deprecated / inert.** Formerly the model identifier for managed + /// (`llm_backend = "cloud"`) summarization. The managed summarization tier is + /// now fixed at `summarization-v1` + /// (`inference::provider::factory::summarization_tier_model`) + /// and this field is no longer consumed — the hosted backend serves exactly + /// one tier for this workload. Kept for config back-compat (existing + /// `config.toml` / `OPENHUMAN_MEMORY_TREE_CLOUD_LLM_MODEL` still parse without + /// error). To run summarization on a different model, point `memory_provider` + /// at a BYOK/local provider instead, where the model rides in the provider + /// string. + /// + /// Defaults to [`DEFAULT_CLOUD_LLM_MODEL`] (`summarization-v1`). + #[serde(default = "default_cloud_llm_model")] + pub cloud_llm_model: Option, + + /// Provider:model string for the smart_walk retrieval agent (e.g. + /// `"deepseek:deepseek-chat"`). When set, the smart walk loop uses this + /// model instead of the general memory/chat provider. Fast, cheap models + /// work best here since the walker makes many short-turn calls. + /// + /// Env override: `OPENHUMAN_MEMORY_TREE_SMART_WALK_MODEL`. + #[serde(default)] + pub smart_walk_model: Option, + + /// Explicit opt-in to cloud-based summarization when local AI is disabled. + /// + /// Default `false` — "Build Summary Trees" was local-only before #002. + /// Enabling this routes workspace memory summaries to the configured cloud + /// provider. Set `memory_tree.cloud_summarization_opt_in = true` or + /// `OPENHUMAN_MEMORY_TREE_CLOUD_SUMMARIZATION=true` to acknowledge that memory + /// content will be sent to an external service. + #[serde(default)] + pub cloud_summarization_opt_in: bool, + + /// Enable the spaCy NER sidecar used by the deterministic (E2GraphRAG) + /// retriever to extract entities from a query. When `true` (default), the + /// managed Python runtime provisions spaCy on first use and serves entity + /// extraction over stdio. When `false` — or whenever Python/spaCy is + /// unavailable — query-entity extraction falls back to the in-Rust + /// regex+LLM extractor (`score::extract`). Env override: + /// `OPENHUMAN_MEMORY_TREE_SPACY_ENABLED`. + #[serde(default = "default_memory_tree_spacy_enabled")] + pub spacy_enabled: bool, +} + +fn default_memory_tree_spacy_enabled() -> bool { + // Opt-in (#5056). Default OFF so a fresh install never provisions the spaCy + // venv + `en_core_web_sm` model on first launch, and the runtime Python + // server is not spawned on every boot when no local NLP is configured. + // Query-entity extraction degrades to the in-Rust regex+LLM extractor + // (`score::extract`); operators opt in via config or + // `OPENHUMAN_MEMORY_TREE_SPACY_ENABLED=1`. + false +} + +/// Returns `None` so that existing installs that never opted into Phase 4 +/// embeddings stay on the inert zero-vector path rather than suddenly +/// attempting to reach a local Ollama daemon they haven't configured. +/// Operators enable the Ollama path by setting either `embedding_endpoint` +/// in TOML or the `OPENHUMAN_MEMORY_EMBED_ENDPOINT` env var. +fn default_memory_tree_embedding_endpoint() -> Option { + None +} + +fn default_memory_tree_embedding_model() -> Option { + None +} + +fn default_memory_tree_embedding_timeout_ms() -> Option { + Some(10_000) +} + +/// Defaults to `false` so installs without an embedding endpoint fall back +/// to the inert zero-vector embedder (with a warn log) instead of refusing +/// to run. Set to `true` in production configs that require embeddings. +fn default_memory_tree_embedding_strict() -> bool { + false +} + +/// Shared `None` default for the LLM-path fields (extractor + summariser +/// endpoints + models). Keeping the same function for all of them makes +/// the intent explicit. +/// +/// Default points at the standard Ollama localhost listener. A user +/// who sets `llm_backend = "local"` plus a `_model` is clearly opting +/// into Ollama, and forcing them to also specify the endpoint just to +/// hit `localhost:11434` was a stealth foot-gun: the +/// `OllamaChatProvider` returned an error on an empty endpoint, which +/// the summariser silently swallowed into its `InertSummariser` +/// fallback — producing concat-and-truncate "summaries" that looked +/// correct but didn't run any LLM at all. With a default endpoint in +/// place, the only signal needed to enable a local LLM seal is a +/// non-empty `_model`. Override via TOML or +/// `OPENHUMAN_MEMORY_TREE_LLM_*_ENDPOINT` to point at a different +/// Ollama host. +fn default_memory_tree_llm_endpoint() -> Option { + Some("http://localhost:11434".to_string()) +} + +fn default_memory_tree_llm_extractor_timeout_ms() -> Option { + Some(15_000) +} + +fn default_memory_tree_llm_summariser_timeout_ms() -> Option { + // 120s — large enough for small/medium local models to finish a + // seal-budget summary on a cold-loaded weight cache. Tighter + // values cause the LlmSummariser to time out and silently fall + // back to InertSummariser (no LLM signal in the resulting node). + Some(120_000) +} + +/// Returns `None` so the default `/memory_tree/content/` path is +/// used unless explicitly overridden via TOML or env var. +fn default_memory_tree_content_dir() -> Option { + None +} + +/// Default Ollama model for the memory-tree LLMs (extractor + summariser). +/// +/// `gemma3:4b` is in the Gemma 3 family (Gemma 4 isn't released yet) +/// and sits between the 1B compact tier and the 12B/27B large tiers. +/// At ~3 GB on disk and ~8 GB RAM at inference it stays inside the +/// envelope of a typical laptop and produces coherent abstractive +/// summaries on real Gmail inboxes — smaller models (≤1.5B) regress +/// to "the email says X, the email says Y" enumeration that's barely +/// better than the InertSummariser concat fallback. +/// +/// Override via `memory_tree.llm_summariser_model` / +/// `llm_extractor_model` in TOML (or `OPENHUMAN_MEMORY_TREE_LLM_*_MODEL` +/// env vars) to scale up (`gemma3:12b-it-qat`, `llama3.1:8b`) or down +/// (`gemma3:1b-it-qat`) for the host's headroom. The frontend +/// `ModelCatalog` lists the curated picks the UI offers as +/// downloadable presets. +fn default_memory_tree_llm_model() -> Option { + Some("gemma3:4b".to_string()) +} + +impl Default for MemoryTreeConfig { + fn default() -> Self { + Self { + embedding_endpoint: default_memory_tree_embedding_endpoint(), + embedding_model: default_memory_tree_embedding_model(), + embedding_timeout_ms: default_memory_tree_embedding_timeout_ms(), + embedding_strict: default_memory_tree_embedding_strict(), + llm_extractor_endpoint: default_memory_tree_llm_endpoint(), + llm_extractor_model: default_memory_tree_llm_model(), + llm_extractor_timeout_ms: default_memory_tree_llm_extractor_timeout_ms(), + llm_summariser_endpoint: default_memory_tree_llm_endpoint(), + llm_summariser_model: default_memory_tree_llm_model(), + llm_summariser_timeout_ms: default_memory_tree_llm_summariser_timeout_ms(), + content_dir: default_memory_tree_content_dir(), + llm_backend: default_llm_backend(), + cloud_llm_model: default_cloud_llm_model(), + smart_walk_model: None, + cloud_summarization_opt_in: false, + spacy_enabled: default_memory_tree_spacy_enabled(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn llm_default_is_cloud() { + assert_eq!(LlmBackend::default(), LlmBackend::Cloud); + assert_eq!(MemoryTreeConfig::default().llm_backend, LlmBackend::Cloud); + } + + #[test] + fn llm_round_trip() { + for v in [LlmBackend::Cloud, LlmBackend::Local] { + assert_eq!(LlmBackend::parse(v.as_str()).unwrap(), v); + } + } + + #[test] + fn llm_parse_is_case_insensitive() { + assert_eq!(LlmBackend::parse("CLOUD").unwrap(), LlmBackend::Cloud); + assert_eq!(LlmBackend::parse(" Local ").unwrap(), LlmBackend::Local); + } + + #[test] + fn llm_parse_rejects_unknown() { + assert!(LlmBackend::parse("hybrid").is_err()); + assert!(LlmBackend::parse("").is_err()); + } + + #[test] + fn cloud_llm_model_default_is_summarizer_v1() { + let cfg = MemoryTreeConfig::default(); + assert_eq!( + cfg.cloud_llm_model.as_deref(), + Some(DEFAULT_CLOUD_LLM_MODEL) + ); + assert_eq!(DEFAULT_CLOUD_LLM_MODEL, "summarization-v1"); + } + + /// #5056: spaCy is opt-in — a fresh install must never provision the + /// spaCy venv / `en_core_web_sm` model, nor spawn the runtime Python + /// server, without an explicit config or env-var opt-in. + #[test] + fn spacy_enabled_defaults_to_false() { + assert!(!MemoryTreeConfig::default().spacy_enabled); + assert!(!default_memory_tree_spacy_enabled()); + } + + #[test] + fn memory_tree_config_default_content_dir_is_none() { + let cfg = MemoryTreeConfig::default(); + assert!( + cfg.content_dir.is_none(), + "default content_dir must be None so workspace default path is used" + ); + } + + /// Verify that the env-var override logic correctly maps non-empty strings + /// to `Some(PathBuf)` and empty/blank strings to `None`. We test the + /// logic inline (not via `apply_env_overrides`) to avoid mutating the + /// process environment in a way that could race with parallel tests. + #[test] + fn content_dir_env_override_logic() { + // Simulate the load.rs overlay logic. + let apply = |raw: &str| -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + None + } else { + Some(PathBuf::from(trimmed)) + } + }; + + assert_eq!(apply("/tmp/foo"), Some(PathBuf::from("/tmp/foo"))); + assert_eq!(apply(" /tmp/foo "), Some(PathBuf::from("/tmp/foo"))); + assert_eq!(apply(""), None); + assert_eq!(apply(" "), None); + } +} diff --git a/src/openhuman/memory/api/host/subsystems.rs b/src/openhuman/memory/api/host/subsystems.rs new file mode 100644 index 0000000000..c68a9241df --- /dev/null +++ b/src/openhuman/memory/api/host/subsystems.rs @@ -0,0 +1,261 @@ +//! `[subsystems.*]` config section — the uniform cross-subsystem driver-binding +//! shape defined in `docs/specs/kernel.md` §3.6 and `docs/specs/plan-memory.md` §4.5. +//! +//! GREENFIELD / ZERO BEHAVIOUR CHANGE: nothing reads this config yet. It exists +//! so `[subsystems.memory]` can be authored today and so `inference`, +//! `channels`, `sandbox`, … can slot in later as sibling fields on +//! [`SubsystemsConfig`] without reshaping this type. +//! +//! Shape (kernel.md §3.6 / plan-memory.md §4.5): +//! +//! ```toml +//! [subsystems.memory] +//! driver = "tinymemory" +//! +//! [subsystems.memory.hooks] +//! auto_recall = true +//! auto_capture = true +//! max_context_tokens = 2000 +//! recall_max_chars = 1000 +//! capture_max_chars = 500 +//! +//! [subsystems.memory.drivers.supermemory] +//! class = "external" +//! transport = "http" +//! endpoint = "https://api.supermemory.ai" +//! credential_ref = "keychain:supermemory" +//! trust_state = "untrusted" +//! ``` + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Top-level `[subsystems]` config block. Currently carries only `memory`; +/// future subsystems (`inference`, `channels`, `sandbox`, …) are added here +/// as sibling fields — see kernel.md §3.6. +#[derive(Debug, Clone, Serialize, Deserialize, Default, JsonSchema)] +#[serde(default)] +pub struct SubsystemsConfig { + #[serde(default)] + pub memory: MemorySubsystemConfig, +} + +/// `[subsystems.memory]` — which driver is bound for the memory subsystem, +/// its hook budgets, and the per-driver option table. +/// +/// `PartialEq`/`Eq` let `CoreContext::rebind_workspace` short-circuit a +/// no-op rebind by comparing the config it was handed against the one already +/// held — equality is value comparison only, so it never prints or leaks the +/// credential fields the way `Debug` would. `Hash` lets `binding` +/// key its per-workspace cache on the whole config, so a changed driver/hooks/ +/// trust for an already-bound workspace yields a fresh binding rather than a +/// stale cache hit. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemorySubsystemConfig { + /// The bound driver id (e.g. `"tinymemory"`, `"supermemory"`, `"null"`). + /// Must match a key under `drivers` when that driver needs options. + #[serde(default = "default_memory_driver")] + pub driver: String, + + #[serde(default)] + pub hooks: MemoryHooksConfig, + + /// Per-driver option tables, keyed by driver id. The module default + /// (`tinymemory`) needs no entry here — its options continue to live in + /// the existing `[memory]` / `[memory_tree]` / `[[memory_sources]]` + /// blocks (plan-memory.md §4.5: "no user-visible config break"). + #[serde(default)] + pub drivers: BTreeMap, +} + +fn default_memory_driver() -> String { + "tinymemory".into() +} + +impl Default for MemorySubsystemConfig { + fn default() -> Self { + Self { + driver: default_memory_driver(), + hooks: MemoryHooksConfig::default(), + drivers: BTreeMap::new(), + } + } +} + +/// Memory-hook budgets — the auto-recall / auto-capture behavior gating +/// values. Defaults reproduce today's (pre-`[subsystems]`) behavior exactly; +/// nothing reads these yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemoryHooksConfig { + #[serde(default = "default_true")] + pub auto_recall: bool, + #[serde(default = "default_true")] + pub auto_capture: bool, + #[serde(default = "default_max_context_tokens")] + pub max_context_tokens: usize, + #[serde(default = "default_recall_max_chars")] + pub recall_max_chars: usize, + #[serde(default = "default_capture_max_chars")] + pub capture_max_chars: usize, +} + +fn default_true() -> bool { + true +} +fn default_max_context_tokens() -> usize { + 2000 +} +fn default_recall_max_chars() -> usize { + 1000 +} +fn default_capture_max_chars() -> usize { + 500 +} + +impl Default for MemoryHooksConfig { + fn default() -> Self { + Self { + auto_recall: default_true(), + auto_capture: default_true(), + max_context_tokens: default_max_context_tokens(), + recall_max_chars: default_recall_max_chars(), + capture_max_chars: default_capture_max_chars(), + } + } +} + +/// One entry under `[subsystems.memory.drivers.]`. Describes an +/// external/embedded driver binding — class, transport, endpoint, and a +/// *reference* to a credential resolved via the keychain (never an inline +/// secret; plan-memory.md §4.5, kernel.md §3.6). +/// +/// `trust_state` is fail-closed `"untrusted"` per kernel.md §3.4: an external +/// driver must have its trust explicitly raised before bind succeeds. +/// +/// MUST NOT derive `Debug` — see the manual impl below. `credential_ref` is a +/// secret handle and plan-memory.md §7 Tier-3 conformance requires "credential never +/// in `Debug`/error output", mirroring `storage_memory::MemoryConfig`'s +/// manual redacting `Debug` impl for `agentmemory_secret`. +/// +/// `PartialEq`/`Eq` are safe to derive: they compare values for equality and +/// never render them, so `credential_ref` stays out of any output. +#[derive(Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemoryDriverConfig { + /// Driver class: `"embedded"` | `"external"` | `"null"`. See kernel.md §3.1. + #[serde(default)] + pub class: Option, + + /// Wire transport for external drivers, e.g. `"http"`. See plan-memory.md §4.2. + #[serde(default)] + pub transport: Option, + + /// Base endpoint URL for external/http drivers. + #[serde(default)] + pub endpoint: Option, + + /// A *reference* to a credential (e.g. `"keychain:supermemory"`), + /// resolved kernel-side through the existing keychain — never an inline + /// secret. Redacted in `Debug`/error output; see the manual `Debug` impl. + #[serde(default)] + pub credential_ref: Option, + + /// Fail-closed trust state for this driver binding. Defaults to + /// `"untrusted"`; must be explicitly raised before an external driver's + /// bind succeeds (kernel.md §3.4). + #[serde(default = "default_trust_state")] + pub trust_state: String, +} + +fn default_trust_state() -> String { + "untrusted".into() +} + +impl Default for MemoryDriverConfig { + fn default() -> Self { + Self { + class: None, + transport: None, + endpoint: None, + credential_ref: None, + trust_state: default_trust_state(), + } + } +} + +// Manual `Debug` implementation that redacts `credential_ref`. Without this, +// any `format!("{cfg:?}")` / `tracing::debug!(?cfg, ...)` / panic message +// capturing a `MemoryDriverConfig` would dump the credential reference +// verbatim. The value itself (e.g. `"keychain:supermemory"`) is only a +// *reference*, not the secret — but plan-memory.md §7 Tier-3 conformance requires it +// never appear in Debug/error output regardless, so this mirrors +// `MemoryConfig`'s `agentmemory_secret` treatment exactly. NEVER derive +// `Debug` on this struct. +impl std::fmt::Debug for MemoryDriverConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryDriverConfig") + .field("class", &self.class) + .field("transport", &self.transport) + .field("endpoint", &self.endpoint) + .field( + "credential_ref", + &self.credential_ref.as_ref().map(|_| ""), + ) + .field("trust_state", &self.trust_state) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn subsystems_config_defaults_reproduce_today_behavior() { + let cfg = SubsystemsConfig::default(); + assert_eq!(cfg.memory.driver, "tinymemory"); + assert!(cfg.memory.hooks.auto_recall); + assert!(cfg.memory.hooks.auto_capture); + assert_eq!(cfg.memory.hooks.max_context_tokens, 2000); + assert_eq!(cfg.memory.hooks.recall_max_chars, 1000); + assert_eq!(cfg.memory.hooks.capture_max_chars, 500); + assert!(cfg.memory.drivers.is_empty()); + } + + #[test] + fn absent_subsystems_block_deserializes_to_default() { + let cfg: SubsystemsConfig = toml::from_str("").expect("empty toml parses"); + assert_eq!( + serde_json::to_value(&cfg).unwrap(), + serde_json::to_value(SubsystemsConfig::default()).unwrap() + ); + } + + #[test] + fn memory_driver_config_debug_never_leaks_credential_ref() { + let driver = MemoryDriverConfig { + class: Some("external".into()), + transport: Some("http".into()), + endpoint: Some("https://api.supermemory.ai".into()), + credential_ref: Some("keychain:supermemory-super-secret-value".into()), + trust_state: "untrusted".into(), + }; + let debug_output = format!("{driver:?}"); + assert!( + !debug_output.contains("keychain:supermemory-super-secret-value"), + "Debug output must never contain the credential_ref value: {debug_output}" + ); + assert!( + debug_output.contains(""), + "Debug output should show a redaction marker: {debug_output}" + ); + } + + #[test] + fn memory_driver_config_default_trust_state_is_untrusted() { + assert_eq!(MemoryDriverConfig::default().trust_state, "untrusted"); + } +} diff --git a/src/openhuman/memory/api/host/test_support.rs b/src/openhuman/memory/api/host/test_support.rs new file mode 100644 index 0000000000..63038141fc --- /dev/null +++ b/src/openhuman/memory/api/host/test_support.rs @@ -0,0 +1,209 @@ +//! [`TestHostConfig`] — a concrete, `Default`-able [`MemoryHostConfig`] for +//! tests. +//! +//! `crate::openhuman::memory::core_impl::Config` is `dyn MemoryHostConfig`, which cannot be +//! `Default::default()`ed. The extracted test suites build a config, tweak two +//! or three fields, and pass `&config` into the code under test — a pattern +//! that needs a real struct. This is that struct. +//! +//! It is behind the `test-support` feature and enabled from +//! `tinymemory-core`'s dev-dependencies, so it never enters a shipped build. +//! It is deliberately *not* a mock: the fields are the real config sections +//! with their real serde defaults, so a test that asserts on default behaviour +//! is asserting on the same values production loads. + +use std::path::PathBuf; + +use super::cloud_providers::CloudProviderCreds; +use super::config::{ComposioMode, MemoryHostConfig}; +use super::local_ai::LocalAiConfig; +use super::scheduler_gate::SchedulerGateConfig; +use super::storage_memory::{MemoryConfig, MemoryTreeConfig}; + +/// A concrete host config for tests. Fields are public — mutate them directly +/// rather than reaching for a builder. +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub struct TestHostConfig { + /// See [`MemoryHostConfig::workspace_dir`]. + pub workspace_dir: PathBuf, + /// See [`MemoryHostConfig::config_path`]. + pub config_path: PathBuf, + /// See [`MemoryHostConfig::memory`]. + pub memory: MemoryConfig, + /// See [`MemoryHostConfig::session_token`]. `None` is signed-out. + pub session_token: Option, + /// See [`MemoryHostConfig::memory_tree`]. + pub memory_tree: MemoryTreeConfig, + /// See [`MemoryHostConfig::scheduler_gate`]. + pub scheduler_gate: SchedulerGateConfig, + /// See [`MemoryHostConfig::local_ai`]. + pub local_ai: LocalAiConfig, + /// See [`MemoryHostConfig::cloud_providers`]. + pub cloud_providers: Vec, + /// See [`MemoryHostConfig::embeddings_provider`]. + pub embeddings_provider: Option, + /// See [`MemoryHostConfig::memory_provider`]. + pub memory_provider: Option, + /// See [`MemoryHostConfig::api_url`]. + pub api_url: Option, + /// See [`MemoryHostConfig::default_model`]. + pub default_model: Option, + /// See [`MemoryHostConfig::default_temperature`]. + pub default_temperature: f64, + /// See [`MemoryHostConfig::output_language`]. + pub output_language: Option, + /// See [`MemoryHostConfig::memory_sync_interval_secs`]. + pub memory_sync_interval_secs: Option, + /// See [`MemoryHostConfig::onboarding_completed`]. + pub onboarding_completed: bool, + /// See [`MemoryHostConfig::secrets_encrypt`]. + pub secrets_encrypt: bool, + /// See [`MemoryHostConfig::composio`]. + pub composio: ComposioMode, + /// See [`MemoryHostConfig::memory_sources_json`]. Defaults to an empty + /// array so a test that never touches sources behaves like a fresh install. + pub memory_sources: Option, + /// See [`MemoryHostConfig::composio_source_caps_migration_version`]. + pub composio_source_caps_migration_version: u32, +} + +#[async_trait::async_trait] +impl MemoryHostConfig for TestHostConfig { + fn workspace_dir(&self) -> &PathBuf { + &self.workspace_dir + } + + fn config_path(&self) -> &PathBuf { + &self.config_path + } + + fn memory_tree_content_root(&self) -> PathBuf { + self.memory_tree + .content_dir + .clone() + .unwrap_or_else(|| self.workspace_dir.join("memory_tree").join("content")) + } + + fn memory(&self) -> &MemoryConfig { + &self.memory + } + + fn memory_tree(&self) -> &MemoryTreeConfig { + &self.memory_tree + } + + fn scheduler_gate(&self) -> &SchedulerGateConfig { + &self.scheduler_gate + } + + fn local_ai(&self) -> &LocalAiConfig { + &self.local_ai + } + + fn cloud_providers(&self) -> &Vec { + &self.cloud_providers + } + + fn embeddings_provider(&self) -> Option<&str> { + self.embeddings_provider.as_deref() + } + + fn memory_provider(&self) -> Option<&str> { + self.memory_provider.as_deref() + } + + fn workload_local_model(&self, workload: &str) -> Option { + let raw = match workload { + "memory" => self.memory_provider.as_deref(), + "embeddings" => self.embeddings_provider.as_deref(), + _ => None, + }?; + let model = raw.trim().strip_prefix("ollama:")?.trim(); + if model.is_empty() { + None + } else { + Some(model.to_string()) + } + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn to_arc(&self) -> std::sync::Arc { + std::sync::Arc::new(self.clone()) + } + + fn api_url(&self) -> Option<&str> { + self.api_url.as_deref() + } + + fn effective_backend_api_url(&self) -> String { + // No resolution to do: a test config states its backend URL outright, + // and the host's env/default ladder is not something to reimplement + // here. + self.api_url.clone().unwrap_or_default() + } + + fn session_token(&self) -> Result, String> { + // `Ok(None)` — "read fine, not signed in" — rather than `Err`, so a + // test that never sets a token exercises the signed-out path instead of + // a credential-store failure. + Ok(self.session_token.clone()) + } + + fn default_model(&self) -> Option<&str> { + self.default_model.as_deref() + } + + fn default_temperature(&self) -> f64 { + self.default_temperature + } + + fn output_language(&self) -> Option<&str> { + self.output_language.as_deref() + } + + fn memory_sync_interval_secs(&self) -> Option { + self.memory_sync_interval_secs + } + + fn onboarding_completed(&self) -> bool { + self.onboarding_completed + } + + fn secrets_encrypt(&self) -> bool { + self.secrets_encrypt + } + + fn composio(&self) -> ComposioMode { + self.composio.clone() + } + + fn memory_sources_json(&self) -> anyhow::Result { + Ok(self + .memory_sources + .clone() + .unwrap_or_else(|| serde_json::Value::Array(Vec::new()))) + } + + fn set_memory_sources_json(&mut self, value: serde_json::Value) -> anyhow::Result<()> { + self.memory_sources = Some(value); + Ok(()) + } + + fn composio_source_caps_migration_version(&self) -> u32 { + self.composio_source_caps_migration_version + } + + fn set_composio_source_caps_migration_version(&mut self, version: u32) { + self.composio_source_caps_migration_version = version; + } + + fn apply_env_overrides(&mut self) {} + + async fn save(&self) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/src/openhuman/memory/api/host/usage.rs b/src/openhuman/memory/api/host/usage.rs new file mode 100644 index 0000000000..11d9c45627 --- /dev/null +++ b/src/openhuman/memory/api/host/usage.rs @@ -0,0 +1,33 @@ +//! [`UsageInfo`] — token accounting returned by an inference provider. +//! +//! Lives in the contract crate because both sides name it: the host's chat +//! providers produce it, and the memory subsystem's summariser threads it back +//! out so callers can attribute cost to a summarisation run. It is inert data +//! with no dependencies, so it costs the contract crate nothing. + +/// Token usage information returned by the provider after an inference call. +#[derive(Debug, Clone, Default)] +pub struct UsageInfo { + /// Number of tokens in the input/prompt. + pub input_tokens: u64, + /// Number of tokens in the output/completion. + pub output_tokens: u64, + /// Total context window size for the model (0 if unknown). + pub context_window: u64, + /// Number of input tokens that were served from the KV cache + /// (returned by backends that support prompt caching, e.g. via + /// `openhuman.usage.cached_input_tokens` or + /// `prompt_tokens_details.cached_tokens`). + pub cached_input_tokens: u64, + /// Number of input tokens written into a provider prompt/KV cache on this + /// request (cache-creation / cache-write tokens). Distinct from + /// `cached_input_tokens` (cache reads). Zero when the provider does not + /// report a cache-write breakdown. + pub cache_creation_tokens: u64, + /// Number of reasoning/thinking output tokens when the provider exposes + /// them separately from `output_tokens`. Zero when unavailable. + pub reasoning_tokens: u64, + /// Amount billed for this request in USD (from + /// `openhuman.billing.charged_amount_usd`). Zero when unavailable. + pub charged_amount_usd: f64, +} diff --git a/src/openhuman/memory/api/mod.rs b/src/openhuman/memory/api/mod.rs new file mode 100644 index 0000000000..05061a10c5 --- /dev/null +++ b/src/openhuman/memory/api/mod.rs @@ -0,0 +1,83 @@ +//! Stable public contracts for the TinyMemory memory system. +//! +//! This crate holds the value types, error enum, capability vocabulary, and +//! storage trait that memory engines and their embedding hosts compile +//! against. It is engine-neutral on purpose: `tinycortex` is the default +//! embedded engine, not the owner of the contract, and a second engine +//! (`supermemory`, `mem0`, a self-hosted HTTP backend) implements the same +//! traits without either engine learning about the other. +//! It is deliberately dependency-light (serde / serde_json / +//! chrono / sha2 / anyhow / thiserror / async-trait / uuid only) so depending on +//! the contract never drags in SQLite, git2, reqwest, regex, or an async +//! runtime. +//! +//! ## Self-contained by design +//! +//! Nothing here names a host type. A third-party memory driver must be able to +//! depend on this crate alone, and the *generic* subsystem/driver vocabulary of +//! the OpenHuman kernel (`Driver`, `DriverClass`, `SubsystemRegistry`, the +//! policy `Guard`) must not be inherited from a *memory* crate by whichever +//! subsystem is cut over next. So the contract carries its own identity, +//! capability, and health vocabulary, and the host's memory adapter converts at +//! the boundary — see [`health`] for the shape that conversion relies on. +//! +//! Driver *class* (embedded / external / null) is deliberately **absent**: that +//! is a host configuration fact about how a driver was bound, not something a +//! driver reports about itself. +//! +//! ## The TinyCortex engine's historical paths still resolve +//! +//! This contract used to live in the TinyCortex repository as `tinycortex-api`. +//! That crate is now a deprecated re-export of this one, and the engine crate +//! aliases these modules back into their historical paths +//! (`crate::openhuman::memory::engine::{types, error, traits}`, +//! `crate::openhuman::memory::engine::chunks::types`, `crate::openhuman::memory::engine::tree::runtime::types`, +//! `crate::openhuman::memory::engine::tool_memory::types`, `crate::openhuman::memory::engine::goals::types`), +//! so every existing path keeps resolving unchanged. +//! +//! ## Module map +//! +//! - [`types`]: pure data contracts (entries, hits, taint, namespaces). +//! - [`recall`]: the borrowed [`recall::RecallOpts`] and owned, serde-derived +//! [`recall::OwnedRecallOpts`] recall filters (both re-exported from +//! [`types`]). +//! - [`capabilities`]: the thirteen [`capabilities::Capability`] families and +//! the [`capabilities::Capabilities`] set negotiated at bind time. +//! - [`provider`]: the driver contract — [`provider::MemoryProvider`] plus the +//! thirteen capability family traits and the value types they need. +//! - [`null`]: [`null::NullMemoryProvider`], the reference driver a +//! compiled-out or unconfigured memory subsystem binds to. +//! - [`health`]: [`health::MemoryHealth`], the liveness state a driver reports. +//! - [`version`]: [`CONTRACT_VERSION`] and the [`is_compatible`] bind rule. +//! - [`error`]: the typed [`error::MemoryError`] enum and its result alias. +//! - [`traits`]: the [`traits::Memory`] storage-backend trait. +//! - [`chunks`]: the persisted chunk model ([`chunks::Chunk`], [`chunks::Metadata`], +//! [`chunks::SourceRef`], …) and the deterministic [`chunks::chunk_id`]. +//! - [`tree`]: the markdown summary-tree node model ([`tree::TreeNode`], +//! [`tree::NodeLevel`], [`tree::TreeStatus`], …). +//! - [`tool_memory`]: tool-scoped rule contracts ([`tool_memory::ToolMemoryRule`], …). +//! - [`goals`]: the long-term goals document ([`goals::GoalsDoc`], [`goals::GoalItem`]). +//! - [`host`]: the **host seam** — [`host::MemoryHostConfig`], +//! [`host::EmbeddingProvider`], [`host::MemoryEventSink`], and the memory +//! config sections whose serde form is persisted in a host's `config.toml`. +//! - [`wire`]: the error-name table a driver reached over a bus or a socket +//! round-trips [`error::MemoryError`] through. Shared by both ends of every +//! such transport, so the names cannot drift apart. + +pub mod capabilities; +pub mod chunks; +pub mod error; +pub mod goals; +pub mod health; +pub mod host; +pub mod null; +pub mod provider; +pub mod recall; +pub mod tool_memory; +pub mod traits; +pub mod tree; +pub mod types; +pub mod version; +pub mod wire; + +pub use version::{is_compatible, CONTRACT_VERSION}; diff --git a/src/openhuman/memory/api/null.rs b/src/openhuman/memory/api/null.rs new file mode 100644 index 0000000000..adc020f57b --- /dev/null +++ b/src/openhuman/memory/api/null.rs @@ -0,0 +1,480 @@ +//! [`NullMemoryProvider`] — the reference driver that stores nothing. +//! +//! ## What it is for +//! +//! A memory subsystem that is compiled out, disabled by configuration, or +//! explicitly bound to `driver = "null"` still has to bind *something*: the +//! kernel's registry holds exactly one driver per slot, and code that reaches +//! the slot must find a value rather than an `Option` it has to unwrap at every +//! call site. This is that value. It replaces the hand-written per-domain +//! `stub.rs` files with one generic answer. +//! +//! It is also the fixture the capability-degradation tests bind: with it in the +//! slot, the ten optional families are unadvertised, so their RPC methods are +//! unregistered and their agent tools are absent — and the core still boots. +//! +//! And it is the existence proof for the mandatory set: if +//! [`crate::openhuman::memory::api::provider::MemoryCore`], [`crate::openhuman::memory::api::provider::MemoryRecall`], and +//! [`crate::openhuman::memory::api::provider::MemoryPortability`] could not be implemented without a +//! storage engine, they would be the wrong three to have made mandatory. +//! +//! ## `/dev/null` semantics, and what that costs +//! +//! Writes are **accepted and discarded**; reads return empty. This mirrors the +//! Unix device the driver is named after, and it is the only behaviour that +//! lets the mandatory three be advertised honestly: a `store` that returned +//! [`crate::openhuman::memory::api::error::MemoryError::Unsupported`] would contradict advertising +//! [`crate::openhuman::memory::api::capabilities::Capability::Core`], and one that returned a hard +//! error would turn every optional auto-capture into a user-visible failure. +//! +//! The cost is real: content written here is gone. That is acceptable for a +//! subsystem the operator turned off, and unacceptable as a fallback for a +//! driver that failed to bind — **that** case falls back to the embedded +//! default, never to this. Do not wire it as a general-purpose failure mode. +//! +//! ## Why it implements all thirteen families but advertises three +//! +//! The ten optional families are implemented and every method returns +//! [`crate::openhuman::memory::api::error::MemoryError::Unsupported`] naming its family, but the +//! `as_*` accessors return `None` and +//! [`crate::openhuman::memory::api::provider::MemoryProvider::capabilities`] lists only the mandatory +//! three. So: +//! +//! - through `&dyn MemoryProvider` — the only way product code sees a driver — +//! an unadvertised family is simply **unreachable**, which is the intended +//! degradation; +//! - through the concrete type, a direct call yields a typed, *named* +//! `Unsupported` error, which is what makes the contract's error mapping +//! testable without writing a second mock. +//! +//! [`crate::openhuman::memory::api::provider::audit_provider`] confirms the two views agree. + +use async_trait::async_trait; + +use crate::openhuman::memory::api::capabilities::{Capabilities, Capability}; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::goals::GoalsDoc; +use crate::openhuman::memory::api::health::MemoryHealth; +use crate::openhuman::memory::api::provider::types::{ + DiffReport, EntityHit, ExportPage, ExportRecord, ImportOutcome, IngestItem, IngestOutcome, + MaintenanceReport, SnapshotRef, SourceItem, SourceScope, +}; +use crate::openhuman::memory::api::provider::{ + MemoryCore, MemoryDiff, MemoryDocuments, MemoryEntities, MemoryGoals, MemoryGraph, + MemoryIngest, MemoryMaintenance, MemoryPortability, MemoryProvider, MemoryRecall, + MemorySourceSink, MemoryToolMemory, MemoryTree, +}; +use crate::openhuman::memory::api::recall::OwnedRecallOpts; +use crate::openhuman::memory::api::tool_memory::ToolMemoryRule; +use crate::openhuman::memory::api::tree::{IngestRequest, QueryResult, TreeStatus}; +use crate::openhuman::memory::api::types::{ + GraphRelationRecord, MemoryCategory, MemoryEntry, MemoryKvRecord, MemoryTaint, + NamespaceDocumentInput, NamespaceRetrievalContext, NamespaceSummary, StoredMemoryDocument, +}; + +/// The [`driver_id`](MemoryProvider::driver_id) this driver reports. +pub const NULL_DRIVER_ID: &str = "null"; + +/// Shorthand for the `Unsupported` error every unadvertised family returns. +fn unsupported(capability: Capability) -> Result { + Err(MemoryError::unsupported(capability)) +} + +/// A driver that accepts every write, discards it, and returns nothing. +/// +/// See the module documentation for what it is for, why writes are silently +/// dropped, and why it implements ten families it does not advertise. +#[derive(Debug, Clone, Copy, Default)] +pub struct NullMemoryProvider; + +impl NullMemoryProvider { + /// Construct the null driver. It holds no state, so every instance is + /// interchangeable. + pub const fn new() -> Self { + Self + } +} + +#[async_trait] +impl MemoryProvider for NullMemoryProvider { + fn driver_id(&self) -> &str { + NULL_DRIVER_ID + } + + /// Exactly the mandatory three. The ten optional families are implemented + /// below but deliberately not advertised, so they stay unreachable through + /// the trait object. + fn capabilities(&self) -> Capabilities { + Capabilities::mandatory() + } + + /// Always [`MemoryHealth::Ready`]: a driver with no backing store has + /// nothing that can be unreachable, and reporting `Degraded` would make + /// every status view of a deliberately-disabled subsystem look broken. + async fn health(&self) -> MemoryHealth { + MemoryHealth::Ready + } + + // The `as_*` accessors are all left at their `None` defaults: nothing + // optional is reachable through the trait object. That absence is the whole + // point of this driver, so overriding any of them would be the bug. +} + +#[async_trait] +impl MemoryCore for NullMemoryProvider { + /// Accepts and discards. See the module docs on `/dev/null` semantics. + async fn store( + &self, + _namespace: &str, + _key: &str, + _content: &str, + _category: MemoryCategory, + _session_id: Option<&str>, + _taint: MemoryTaint, + ) -> Result<(), MemoryError> { + Ok(()) + } + + async fn get(&self, _namespace: &str, _key: &str) -> Result, MemoryError> { + Ok(None) + } + + /// Always `Ok(false)`: nothing was ever stored, so nothing existed to + /// forget. Consistent with the idempotence the family requires. + async fn forget(&self, _namespace: &str, _key: &str) -> Result { + Ok(false) + } + + async fn list( + &self, + _namespace: Option<&str>, + _category: Option<&MemoryCategory>, + _session_id: Option<&str>, + ) -> Result, MemoryError> { + Ok(Vec::new()) + } + + async fn namespaces(&self) -> Result, MemoryError> { + Ok(Vec::new()) + } +} + +#[async_trait] +impl MemoryRecall for NullMemoryProvider { + async fn recall( + &self, + _query: &str, + _limit: usize, + _opts: &OwnedRecallOpts, + _scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + Ok(Vec::new()) + } +} + +#[async_trait] +impl MemoryPortability for NullMemoryProvider { + /// One empty, terminal page: no records and no continuation cursor, so a + /// caller's export loop terminates on the first iteration. + /// + /// This driver never issues a cursor (every page is the first and only + /// page), so any `Some(_)` cursor a caller passes back is necessarily one + /// this driver did not hand out — reject it rather than silently treating + /// it as a valid terminal page. + async fn export_page( + &self, + cursor: Option<&str>, + _limit: usize, + ) -> Result { + if cursor.is_some() { + return Err(MemoryError::Invalid( + "null provider does not issue export cursors".into(), + )); + } + + Ok(ExportPage::default()) + } + + /// Counts every record as skipped rather than imported. Reporting them as + /// imported would tell a migration its data landed somewhere it did not. + async fn import_records( + &self, + records: Vec, + ) -> Result { + Ok(ImportOutcome { + imported: 0, + skipped: u32::try_from(records.len()).unwrap_or(u32::MAX), + failed: 0, + errors: Vec::new(), + }) + } +} + +#[async_trait] +impl MemoryIngest for NullMemoryProvider { + async fn ingest_document(&self, _item: IngestItem) -> Result { + unsupported(Capability::Ingest) + } + + async fn ingest_chat(&self, _messages: Vec) -> Result { + unsupported(Capability::Ingest) + } +} + +#[async_trait] +impl MemoryDocuments for NullMemoryProvider { + async fn put_document(&self, _input: NamespaceDocumentInput) -> Result { + unsupported(Capability::Documents) + } + + async fn get_document( + &self, + _namespace: &str, + _key: &str, + ) -> Result, MemoryError> { + unsupported(Capability::Documents) + } + + async fn list_documents( + &self, + _namespace: Option<&str>, + ) -> Result { + unsupported(Capability::Documents) + } + + async fn list_namespaces(&self) -> Result, MemoryError> { + unsupported(Capability::Documents) + } + + async fn delete_document( + &self, + _namespace: &str, + _document_id: &str, + ) -> Result { + unsupported(Capability::Documents) + } + + async fn clear_namespace(&self, _namespace: &str) -> Result<(), MemoryError> { + unsupported(Capability::Documents) + } + + async fn query_documents( + &self, + _namespace: &str, + _query: &str, + _limit: usize, + ) -> Result { + unsupported(Capability::Documents) + } + + async fn recall_documents( + &self, + _namespace: &str, + _limit: usize, + ) -> Result { + unsupported(Capability::Documents) + } +} + +#[async_trait] +impl MemoryTree for NullMemoryProvider { + async fn append(&self, _request: IngestRequest) -> Result<(), MemoryError> { + unsupported(Capability::Tree) + } + + async fn query_source( + &self, + _namespace: &str, + _source_id: &str, + _limit: usize, + _scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + unsupported(Capability::Tree) + } + + async fn drill_down( + &self, + _namespace: &str, + _node_id: &str, + ) -> Result { + unsupported(Capability::Tree) + } + + async fn seal(&self, _namespace: &str) -> Result { + unsupported(Capability::Tree) + } + + async fn cascade(&self, _namespace: &str) -> Result { + unsupported(Capability::Tree) + } +} + +#[async_trait] +impl MemoryEntities for NullMemoryProvider { + async fn entities( + &self, + _namespace: &str, + _query: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + unsupported(Capability::Entities) + } + + async fn entity_edges( + &self, + _namespace: &str, + _entity_id: &str, + _limit: usize, + ) -> Result, MemoryError> { + unsupported(Capability::Entities) + } + + async fn touch_entities( + &self, + _namespace: &str, + _entity_ids: &[String], + ) -> Result<(), MemoryError> { + unsupported(Capability::Entities) + } +} + +#[async_trait] +impl MemoryGraph for NullMemoryProvider { + async fn kv_get( + &self, + _namespace: Option<&str>, + _key: &str, + ) -> Result, MemoryError> { + unsupported(Capability::Graph) + } + + async fn kv_put( + &self, + _namespace: Option<&str>, + _key: &str, + _value: serde_json::Value, + ) -> Result<(), MemoryError> { + unsupported(Capability::Graph) + } + + async fn kv_delete(&self, _namespace: Option<&str>, _key: &str) -> Result { + unsupported(Capability::Graph) + } + + async fn kv_list( + &self, + _namespace: Option<&str>, + _prefix: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + unsupported(Capability::Graph) + } + + async fn relations( + &self, + _namespace: Option<&str>, + _subject: Option<&str>, + _predicate: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + unsupported(Capability::Graph) + } + + async fn put_relation(&self, _relation: GraphRelationRecord) -> Result<(), MemoryError> { + unsupported(Capability::Graph) + } +} + +#[async_trait] +impl MemoryDiff for NullMemoryProvider { + async fn capture_snapshot(&self, _source_id: &str) -> Result { + unsupported(Capability::Diff) + } + + async fn snapshots( + &self, + _source_id: &str, + _limit: usize, + ) -> Result, MemoryError> { + unsupported(Capability::Diff) + } + + async fn diff( + &self, + _source_id: &str, + _from: Option<&str>, + _to: &str, + ) -> Result { + unsupported(Capability::Diff) + } +} + +#[async_trait] +impl MemoryGoals for NullMemoryProvider { + async fn goals(&self) -> Result { + unsupported(Capability::Goals) + } + + async fn set_goals(&self, _goals: GoalsDoc) -> Result<(), MemoryError> { + unsupported(Capability::Goals) + } +} + +#[async_trait] +impl MemoryToolMemory for NullMemoryProvider { + async fn tool_rules(&self, _tool_name: &str) -> Result, MemoryError> { + unsupported(Capability::ToolMemory) + } + + async fn put_tool_rule(&self, _rule: ToolMemoryRule) -> Result<(), MemoryError> { + unsupported(Capability::ToolMemory) + } + + async fn delete_tool_rule( + &self, + _tool_name: &str, + _rule_id: &str, + ) -> Result { + unsupported(Capability::ToolMemory) + } +} + +#[async_trait] +impl MemorySourceSink for NullMemoryProvider { + async fn accept_source_items( + &self, + _source_id: &str, + _source_kind: &str, + _items: Vec, + _taint: MemoryTaint, + ) -> Result { + unsupported(Capability::Sources) + } + + async fn forget_source(&self, _source_id: &str) -> Result { + unsupported(Capability::Sources) + } +} + +#[async_trait] +impl MemoryMaintenance for NullMemoryProvider { + async fn reembed(&self) -> Result { + unsupported(Capability::Maintenance) + } + + async fn compact(&self) -> Result { + unsupported(Capability::Maintenance) + } + + async fn consolidate(&self) -> Result { + unsupported(Capability::Maintenance) + } + + async fn doctor(&self) -> Result { + unsupported(Capability::Maintenance) + } +} + +#[cfg(test)] +#[path = "null_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/null_tests.rs b/src/openhuman/memory/api/null_tests.rs new file mode 100644 index 0000000000..bd65e8a27c --- /dev/null +++ b/src/openhuman/memory/api/null_tests.rs @@ -0,0 +1,242 @@ +//! Tests for the reference null driver. +//! +//! These pin three separate contracts: +//! +//! 1. the mandatory-three set is genuinely implementable without a store; +//! 2. an unadvertised family is **unreachable** through the trait object, which +//! is the degradation behaviour the kernel relies on; +//! 3. a direct call to an unadvertised family yields a typed `Unsupported` +//! error that **names** the family, which is what the transport adapter's +//! `501` mapping is checked against. +//! +//! ## No async runtime here, on purpose +//! +//! `tinymemory-api` must not depend on tokio (or any executor) — that is the +//! whole point of the crate. Every future in this module completes on its first +//! poll, so a six-line std-only [`block_on`] is sufficient and adds no +//! dependency. + +use std::future::Future; +use std::pin::pin; +use std::task::{Context, Poll}; + +use super::*; +use crate::openhuman::memory::api::provider::audit_provider; +use crate::openhuman::memory::api::types::MemoryCategory; + +/// Drive a future that is ready on first poll to completion, without an +/// executor. Panics rather than spinning if a future ever returns `Pending`, +/// because in this module that would mean a supposedly-inert implementation +/// started doing real work. +fn block_on(future: F) -> F::Output { + let mut future = pin!(future); + let mut context = Context::from_waker(std::task::Waker::noop()); + match future.as_mut().poll(&mut context) { + Poll::Ready(value) => value, + Poll::Pending => panic!("null driver future must complete on first poll"), + } +} + +#[test] +fn null_driver_advertises_exactly_the_mandatory_families() { + let driver = NullMemoryProvider::new(); + let capabilities = driver.capabilities(); + + assert_eq!(driver.driver_id(), NULL_DRIVER_ID); + assert_eq!(capabilities.len(), 3); + for capability in Capability::MANDATORY { + assert!( + capabilities.contains(capability), + "{capability} must be advertised" + ); + } +} + +#[test] +fn null_driver_passes_capability_validation() { + // The mandatory-three set is the minimum bindable set, so the reference + // driver must be bindable. If this ever fails, either the mandatory list + // grew or the null driver stopped implementing it. + let driver = NullMemoryProvider::new(); + assert_eq!(driver.capabilities().validate(), Ok(())); +} + +#[test] +fn null_driver_is_self_consistent() { + assert_eq!(audit_provider(&NullMemoryProvider::new()), Ok(())); +} + +#[test] +fn null_driver_reports_ready() { + let health = block_on(NullMemoryProvider::new().health()); + assert_eq!(health, MemoryHealth::Ready); + assert!(health.is_usable()); +} + +#[test] +fn null_driver_shutdown_is_an_idempotent_no_op() { + let driver = NullMemoryProvider::new(); + assert!(block_on(driver.shutdown()).is_ok()); + assert!(block_on(driver.shutdown()).is_ok()); +} + +#[test] +fn mandatory_core_accepts_writes_and_reads_back_empty() { + let driver = NullMemoryProvider::new(); + + block_on(driver.store( + "global", + "k", + "v", + MemoryCategory::Core, + None, + MemoryTaint::ExternalSync, + )) + .expect("null store must accept the write"); + + assert!(block_on(driver.get("global", "k")) + .expect("get must succeed") + .is_none()); + assert!(!block_on(driver.forget("global", "k")).expect("forget must succeed")); + assert!(block_on(driver.list(None, None, None)) + .expect("list must succeed") + .is_empty()); + assert!(block_on(driver.namespaces()) + .expect("namespaces must succeed") + .is_empty()); +} + +#[test] +fn mandatory_recall_returns_no_hits() { + let driver = NullMemoryProvider::new(); + let hits = block_on(driver.recall("anything", 10, &OwnedRecallOpts::default(), None)) + .expect("recall must succeed"); + assert!(hits.is_empty()); +} + +#[test] +fn mandatory_portability_round_trips_as_an_empty_store() { + let driver = NullMemoryProvider::new(); + + let page = block_on(driver.export_page(None, 100)).expect("export must succeed"); + assert!(page.records.is_empty()); + assert!( + page.next_cursor.is_none(), + "the absent cursor is what terminates the caller's export loop" + ); + + let outcome = block_on(driver.import_records(vec![ExportRecord { + kind: "entry".to_string(), + id: "rec-1".to_string(), + namespace: None, + taint: MemoryTaint::Internal, + payload: serde_json::Value::Null, + }])) + .expect("import must succeed"); + + // Skipped, never imported: reporting an import would tell a migration its + // data landed somewhere it did not. + assert_eq!(outcome.imported, 0); + assert_eq!(outcome.skipped, 1); + assert_eq!(outcome.failed, 0); +} + +#[test] +fn export_page_rejects_a_cursor_it_never_issued() { + let driver = NullMemoryProvider::new(); + + let err = block_on(driver.export_page(Some("unexpected"), 100)) + .expect_err("a cursor this driver never issued must be rejected, not silently accepted"); + assert!( + matches!(err, MemoryError::Invalid(_)), + "expected MemoryError::Invalid, got {err:?}" + ); +} + +#[test] +fn every_unadvertised_family_is_unreachable_through_the_trait_object() { + let driver = NullMemoryProvider::new(); + let provider: &dyn MemoryProvider = &driver; + + assert!(provider.as_ingest().is_none()); + assert!(provider.as_documents().is_none()); + assert!(provider.as_tree().is_none()); + assert!(provider.as_entities().is_none()); + assert!(provider.as_graph().is_none()); + assert!(provider.as_diff().is_none()); + assert!(provider.as_goals().is_none()); + assert!(provider.as_tool_memory().is_none()); + assert!(provider.as_sources().is_none()); + assert!(provider.as_maintenance().is_none()); +} + +#[test] +fn advertised_and_reachable_agree_for_every_family() { + // The invariant that keeps the capability set honest, checked family by + // family rather than only through the aggregate audit. + let driver = NullMemoryProvider::new(); + let provider: &dyn MemoryProvider = &driver; + let advertised = provider.capabilities(); + + for capability in Capability::ALL { + assert_eq!( + advertised.contains(capability), + provider.provides(capability), + "{capability}: advertised and reachable must agree" + ); + } +} + +/// Assert a result is `Unsupported` and names the expected family. +fn assert_unsupported(result: Result, expected: Capability) { + match result { + Err(MemoryError::Unsupported { capability }) => { + assert_eq!(capability, expected.as_str()); + } + other => panic!("expected Unsupported({expected}), got {other:?}"), + } +} + +#[test] +fn unadvertised_families_return_unsupported_naming_their_capability() { + let driver = NullMemoryProvider::new(); + + assert_unsupported(block_on(driver.ingest_chat(Vec::new())), Capability::Ingest); + assert_unsupported( + block_on(driver.get_document("global", "k")), + Capability::Documents, + ); + assert_unsupported(block_on(driver.seal("global")), Capability::Tree); + assert_unsupported( + block_on(driver.entities("global", None, 10)), + Capability::Entities, + ); + assert_unsupported(block_on(driver.kv_get(None, "k")), Capability::Graph); + assert_unsupported( + block_on(driver.capture_snapshot("src-abc")), + Capability::Diff, + ); + assert_unsupported(block_on(driver.goals()), Capability::Goals); + assert_unsupported(block_on(driver.tool_rules("shell")), Capability::ToolMemory); + assert_unsupported( + block_on(driver.forget_source("src-abc")), + Capability::Sources, + ); + assert_unsupported(block_on(driver.doctor()), Capability::Maintenance); +} + +#[test] +fn provider_is_usable_as_a_shared_trait_object() { + // The registry binds `Arc`, so the trait object must be + // `Send + Sync` and every family trait must be object-safe. This test fails + // to *compile* rather than to run if that ever regresses. + fn assert_send_sync(_value: &T) {} + + let provider: std::sync::Arc = + std::sync::Arc::new(NullMemoryProvider::new()); + assert_send_sync(&provider); + assert_eq!(provider.driver_id(), NULL_DRIVER_ID); + assert!(block_on(provider.list(None, None, None)) + .expect("list through the trait object") + .is_empty()); +} diff --git a/src/openhuman/memory/api/provider/audit.rs b/src/openhuman/memory/api/provider/audit.rs new file mode 100644 index 0000000000..aa0d9980fb --- /dev/null +++ b/src/openhuman/memory/api/provider/audit.rs @@ -0,0 +1,132 @@ +//! The honesty check: does a driver's advertised capability set match the +//! surface it actually exposes? +//! +//! [`MemoryProvider::capabilities`] is a *claim*, and the kernel acts on it — +//! it registers RPC methods and assembles agent tools from the advertised set +//! and never re-checks. A driver that advertises a family it does not implement +//! therefore produces a surface that exists in `/schema`, appears in the agent's +//! tool list, and fails on first use. That is precisely the +//! "registered-but-failing" outcome the degradation design exists to avoid. +//! +//! [`audit_provider`] compares the claim against +//! [`MemoryProvider::provides`] — which is derived from the accessors, so it +//! cannot drift from reality — and reports both directions of mismatch. Run it +//! at bind time next to [`crate::openhuman::memory::api::capabilities::Capabilities::validate`], and in +//! every driver's own test suite. +//! +//! The two directions mean different things: +//! +//! - **Advertised but absent** is a bug that will surface as a failing call. It +//! should refuse the bind. +//! - **Present but unadvertised** is dead surface: the family works but the +//! kernel unregistered it, so nothing can reach it. Usually a forgotten +//! entry in the driver's `capabilities()` list. + +use std::fmt; + +use crate::openhuman::memory::api::capabilities::Capability; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::provider::driver::MemoryProvider; + +/// A disagreement between what a driver advertises and what it implements. +/// +/// Carries the families structurally rather than as a formatted string so a +/// caller can report them in a status payload or a bind-failure event as well +/// as in a log line. At least one of the two vectors is non-empty. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CapabilityAudit { + /// Families the driver advertises but does not expose. These will fail on + /// first call; refuse the bind. + pub advertised_but_absent: Vec, + /// Families the driver exposes but does not advertise. These are + /// unreachable, because the kernel filters from the advertised set. + pub present_but_unadvertised: Vec, +} + +impl fmt::Display for CapabilityAudit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut parts = Vec::new(); + if !self.advertised_but_absent.is_empty() { + parts.push(format!( + "advertised but not implemented: {}", + join(&self.advertised_but_absent) + )); + } + if !self.present_but_unadvertised.is_empty() { + parts.push(format!( + "implemented but not advertised: {}", + join(&self.present_but_unadvertised) + )); + } + write!(f, "memory driver capability mismatch; {}", parts.join("; ")) + } +} + +impl std::error::Error for CapabilityAudit {} + +impl From for MemoryError { + /// A mismatch is the driver saying something untrue about itself, which is + /// a configuration/implementation error rather than an unsupported call — + /// hence [`MemoryError::Invalid`] and not + /// [`MemoryError::Unsupported`]. Same reasoning as + /// [`crate::openhuman::memory::api::capabilities::MissingMandatoryCapabilities`]. + fn from(value: CapabilityAudit) -> Self { + MemoryError::Invalid(value.to_string()) + } +} + +fn join(families: &[Capability]) -> String { + families + .iter() + .map(|cap| cap.as_str()) + .collect::>() + .join(", ") +} + +/// Compare a driver's advertised capability set against its reachable surface. +/// +/// Walks every [`Capability`] in declaration order, so the returned vectors are +/// in that order too. +/// +/// # Errors +/// +/// Returns [`CapabilityAudit`] when the two disagree in either direction. A +/// driver that agrees with itself returns `Ok(())`. +/// +/// # Examples +/// +/// ``` +/// # use crate::openhuman::memory::api::null::NullMemoryProvider; +/// # use crate::openhuman::memory::api::provider::audit_provider; +/// // The reference null driver is self-consistent. +/// assert!(audit_provider(&NullMemoryProvider::new()).is_ok()); +/// ``` +pub fn audit_provider(provider: &dyn MemoryProvider) -> Result<(), CapabilityAudit> { + let advertised = provider.capabilities(); + let mut advertised_but_absent = Vec::new(); + let mut present_but_unadvertised = Vec::new(); + + for capability in Capability::ALL { + match ( + advertised.contains(capability), + provider.provides(capability), + ) { + (true, false) => advertised_but_absent.push(capability), + (false, true) => present_but_unadvertised.push(capability), + _ => {} + } + } + + if advertised_but_absent.is_empty() && present_but_unadvertised.is_empty() { + Ok(()) + } else { + Err(CapabilityAudit { + advertised_but_absent, + present_but_unadvertised, + }) + } +} + +#[cfg(test)] +#[path = "audit_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/provider/audit_tests.rs b/src/openhuman/memory/api/provider/audit_tests.rs new file mode 100644 index 0000000000..814f1597c5 --- /dev/null +++ b/src/openhuman/memory/api/provider/audit_tests.rs @@ -0,0 +1,201 @@ +//! Tests for the advertised-vs-implemented honesty check. +//! +//! Two deliberately dishonest fixtures sit here — one that over-claims and one +//! that under-claims — because the whole value of [`audit_provider`] is +//! catching drivers that disagree with themselves, and neither direction is +//! reachable from an honest driver. + +use async_trait::async_trait; + +use super::*; +use crate::openhuman::memory::api::capabilities::Capabilities; +use crate::openhuman::memory::api::health::MemoryHealth; +use crate::openhuman::memory::api::null::NullMemoryProvider; +use crate::openhuman::memory::api::provider::types::{ + ExportPage, ExportRecord, ImportOutcome, SourceScope, +}; +use crate::openhuman::memory::api::provider::{ + MemoryCore, MemoryPortability, MemoryRecall, MemoryTree, +}; +use crate::openhuman::memory::api::recall::OwnedRecallOpts; +use crate::openhuman::memory::api::types::{ + MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, +}; + +/// A provider that forwards the mandatory three to [`NullMemoryProvider`] so +/// each fixture below only has to describe the thing it is lying about. +struct Fixture { + inner: NullMemoryProvider, + advertised: Capabilities, + expose_tree: bool, +} + +impl Fixture { + fn new(advertised: Capabilities, expose_tree: bool) -> Self { + Self { + inner: NullMemoryProvider::new(), + advertised, + expose_tree, + } + } +} + +#[async_trait] +impl MemoryCore for Fixture { + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> Result<(), MemoryError> { + self.inner + .store(namespace, key, content, category, session_id, taint) + .await + } + + async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError> { + self.inner.get(namespace, key).await + } + + async fn forget(&self, namespace: &str, key: &str) -> Result { + self.inner.forget(namespace, key).await + } + + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> Result, MemoryError> { + self.inner.list(namespace, category, session_id).await + } + + async fn namespaces(&self) -> Result, MemoryError> { + self.inner.namespaces().await + } +} + +#[async_trait] +impl MemoryRecall for Fixture { + async fn recall( + &self, + query: &str, + limit: usize, + opts: &OwnedRecallOpts, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + self.inner.recall(query, limit, opts, scope).await + } +} + +#[async_trait] +impl MemoryPortability for Fixture { + async fn export_page( + &self, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.inner.export_page(cursor, limit).await + } + + async fn import_records( + &self, + records: Vec, + ) -> Result { + self.inner.import_records(records).await + } +} + +#[async_trait] +impl MemoryProvider for Fixture { + fn driver_id(&self) -> &str { + "fixture" + } + + fn capabilities(&self) -> Capabilities { + self.advertised + } + + async fn health(&self) -> MemoryHealth { + MemoryHealth::Ready + } + + fn as_tree(&self) -> Option<&dyn MemoryTree> { + if self.expose_tree { + Some(&self.inner) + } else { + None + } + } +} + +#[test] +fn honest_driver_passes_the_audit() { + let honest = Fixture::new(Capabilities::mandatory().with(Capability::Tree), true); + assert_eq!(audit_provider(&honest), Ok(())); +} + +#[test] +fn over_claiming_driver_is_reported_as_advertised_but_absent() { + // Advertises everything, exposes no optional accessor. Every one of the ten + // optional families would fail on first call — the exact + // registered-but-failing outcome the capability filter exists to prevent. + let liar = Fixture::new(Capabilities::all(), false); + + let audit = audit_provider(&liar).expect_err("over-claiming driver must fail the audit"); + assert_eq!(audit.present_but_unadvertised, Vec::new()); + assert_eq!(audit.advertised_but_absent.len(), 10); + assert!(audit.advertised_but_absent.contains(&Capability::Tree)); + // The mandatory three are supertraits, so they can never be missing. + assert!(!audit.advertised_but_absent.contains(&Capability::Core)); + assert!(!audit.advertised_but_absent.contains(&Capability::Recall)); + assert!(!audit + .advertised_but_absent + .contains(&Capability::Portability)); +} + +#[test] +fn under_claiming_driver_is_reported_as_present_but_unadvertised() { + // Implements the tree but forgot to list it: the family works and is + // completely unreachable, because the kernel filters from the advertised + // set. + let shy = Fixture::new(Capabilities::mandatory(), true); + + let audit = audit_provider(­).expect_err("under-claiming driver must fail the audit"); + assert_eq!(audit.advertised_but_absent, Vec::new()); + assert_eq!(audit.present_but_unadvertised, vec![Capability::Tree]); +} + +#[test] +fn audit_findings_are_reported_in_declaration_order() { + let liar = Fixture::new(Capabilities::all(), false); + let audit = audit_provider(&liar).expect_err("expected a mismatch"); + + let declaration_order: Vec = Capability::ALL + .into_iter() + .filter(|cap| audit.advertised_but_absent.contains(cap)) + .collect(); + assert_eq!(audit.advertised_but_absent, declaration_order); +} + +#[test] +fn audit_error_names_every_mismatched_family_and_maps_to_invalid() { + let liar = Fixture::new(Capabilities::all(), false); + let audit = audit_provider(&liar).expect_err("expected a mismatch"); + + let rendered = audit.to_string(); + for capability in &audit.advertised_but_absent { + assert!( + rendered.contains(capability.as_str()), + "audit message must name {capability}: {rendered}" + ); + } + + // A driver lying about itself is a config/implementation error, not an + // unsupported call. + let error: MemoryError = audit.into(); + assert!(matches!(error, MemoryError::Invalid(_))); +} diff --git a/src/openhuman/memory/api/provider/content.rs b/src/openhuman/memory/api/provider/content.rs new file mode 100644 index 0000000000..1c1974367e --- /dev/null +++ b/src/openhuman/memory/api/provider/content.rs @@ -0,0 +1,208 @@ +//! Optional families that put content *into* memory and navigate it: +//! [`MemoryIngest`], [`MemoryDocuments`], and [`MemoryTree`]. +//! +//! All three are optional. A driver that advertises none of them is still a +//! memory backend — it just accepts entries only through +//! [`crate::openhuman::memory::api::provider::MemoryCore::store`] and has no document tier and no +//! summary tree. The kernel unregisters the matching RPC methods and omits the +//! matching agent tools rather than registering handlers that fail. +//! +//! ## No configuration crosses this boundary +//! +//! Chunk sizes, embedding models, summariser prompts, seal thresholds, and +//! cascade policy are all *driver* concerns. None of them appear in these +//! signatures: the embedded driver reads them from the `MemoryConfig` it +//! already holds, and an external driver has its own. This was the sharpest +//! test of whether the M0 crate carve-out drew the line in the right place — +//! the families that looked most config-dependent turned out not to need any. + +use async_trait::async_trait; + +use crate::openhuman::memory::api::capabilities::Capability; +use crate::openhuman::memory::api::chunks::Chunk; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::provider::types::{IngestItem, IngestOutcome, SourceScope}; +use crate::openhuman::memory::api::tree::{IngestRequest, QueryResult, TreeStatus}; +use crate::openhuman::memory::api::types::{ + NamespaceDocumentInput, NamespaceRetrievalContext, StoredMemoryDocument, +}; + +/// Bulk content ingestion — the driver owns chunking and embedding. +/// +/// The distinction from [`crate::openhuman::memory::api::provider::MemoryCore::store`] is ownership of +/// the pipeline: `store` persists exactly one entry the caller has already +/// shaped, whereas ingest hands over raw source material and lets the driver +/// decide how to split, embed, and index it. +#[async_trait] +pub trait MemoryIngest: Send + Sync { + /// Ingest one standalone document. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for content the driver refuses (empty body, + /// unsupported MIME), otherwise backend failures. + async fn ingest_document(&self, item: IngestItem) -> Result; + + /// Ingest a run of chat messages that share a conversation. + /// + /// Taken as a batch rather than one call per message because chat chunking + /// is inherently cross-message: a driver needs neighbouring turns to decide + /// where a chunk boundary belongs. Ordering within `messages` is + /// significant and must be preserved by the caller. + /// + /// # Errors + /// + /// As [`Self::ingest_document`]. Partial success is reported through the + /// counts in [`IngestOutcome`], not as an error. + async fn ingest_chat(&self, messages: Vec) -> Result; +} + +/// The namespace-document tier: whole documents addressed by `(namespace, key)`. +/// +/// Distinct from [`crate::openhuman::memory::api::provider::MemoryCore`] in granularity and in what is +/// stored: entries are short facts, documents are bodies with titles, tags, +/// source types, and structured metadata, and they carry their own ranked query +/// surface. +#[async_trait] +pub trait MemoryDocuments: Send + Sync { + /// Upsert a document, returning its driver-assigned id. + /// + /// Keyed by `(namespace, key)` from the input: reusing a key replaces the + /// existing document rather than creating a second one. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a rejected input, otherwise backend + /// failures. + async fn put_document(&self, input: NamespaceDocumentInput) -> Result; + + /// Fetch a document by `(namespace, key)`. + /// + /// # Errors + /// + /// A missing document is `Ok(None)`; `Err` is reserved for backend + /// failures. + async fn get_document( + &self, + namespace: &str, + key: &str, + ) -> Result, MemoryError>; + + /// List document summaries, optionally restricted to one namespace. + async fn list_documents( + &self, + namespace: Option<&str>, + ) -> Result; + + /// List every namespace containing documents. + async fn list_namespaces(&self) -> Result, MemoryError>; + + /// Delete a document by its driver-assigned id. + async fn delete_document( + &self, + namespace: &str, + document_id: &str, + ) -> Result; + + /// Delete all data belonging to one namespace. + async fn clear_namespace(&self, namespace: &str) -> Result<(), MemoryError>; + + /// Run a ranked query over one namespace's documents. + /// + /// Returns both the ranked hits and the driver's rendered context text, so + /// a caller that only wants something injectable does not have to + /// re-assemble it (and re-assemble it differently from every other caller). + /// + /// # Errors + /// + /// Backend failures only; a query that matches nothing returns an empty + /// hit list. + async fn query_documents( + &self, + namespace: &str, + query: &str, + limit: usize, + ) -> Result; + + /// Recall the highest-ranked context from a namespace without a query. + /// + /// This is a distinct engine operation rather than a query with an empty + /// string: query-less recall applies the namespace's freshness and + /// priority ranking without introducing a synthetic search term. + /// + /// # Errors + /// + /// [`MemoryError::Unsupported`] when a provider predating this optional + /// operation does not implement it, otherwise backend failures. An empty + /// namespace returns empty context. + async fn recall_documents( + &self, + _namespace: &str, + _limit: usize, + ) -> Result { + Err(MemoryError::unsupported(Capability::Documents)) + } +} + +/// The time-ordered summary tree: buffered leaves rolled up into hour → day → +/// month → year → root summaries. +/// +/// Sealing and cascading are exposed as explicit calls rather than happening +/// implicitly on ingest because the **host** owns scheduling. A driver runs one +/// step when asked; it does not get to install its own background loop. This is +/// the same rule as the engine's `queue::run_once`. +#[async_trait] +pub trait MemoryTree: Send + Sync { + /// Append raw content to the ingestion buffer for later sealing. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a rejected request, otherwise backend + /// failures. + async fn append(&self, request: IngestRequest) -> Result<(), MemoryError>; + + /// Retrieve the chunks a single logical source contributed, newest first. + /// + /// `scope` is the per-turn allowlist and must be applied **inside** the + /// driver's query, for the reasons in [`SourceScope`]. `None` means + /// unrestricted. + /// + /// # Errors + /// + /// Backend failures only; an unknown `source_id` yields an empty vector. + async fn query_source( + &self, + namespace: &str, + source_id: &str, + limit: usize, + scope: Option<&SourceScope>, + ) -> Result, MemoryError>; + + /// Fetch one node together with its direct children, for navigation. + /// + /// # Errors + /// + /// [`MemoryError::NotFound`] when `node_id` does not exist in `namespace`. + async fn drill_down(&self, namespace: &str, node_id: &str) -> Result; + + /// Convert buffered content into leaf nodes, returning the resulting tree + /// state. + /// + /// Idempotent when the buffer is empty: sealing nothing is a successful + /// no-op, not an error, so a scheduler may call it unconditionally. + /// + /// # Errors + /// + /// Backend failures only. + async fn seal(&self, namespace: &str) -> Result; + + /// Roll sealed leaves up through the parent levels, returning the resulting + /// tree state. + /// + /// Idempotent for the same reason as [`Self::seal`]. + /// + /// # Errors + /// + /// Backend failures only. + async fn cascade(&self, namespace: &str) -> Result; +} diff --git a/src/openhuman/memory/api/provider/driver.rs b/src/openhuman/memory/api/provider/driver.rs new file mode 100644 index 0000000000..f099012bc6 --- /dev/null +++ b/src/openhuman/memory/api/provider/driver.rs @@ -0,0 +1,199 @@ +//! [`MemoryProvider`] — the single trait a memory driver implements, and the +//! object the kernel binds. +//! +//! ## Self-contained on purpose +//! +//! `MemoryProvider` does **not** extend a host `Driver` trait and names no host +//! type. `tinymemory-api` is what a third-party driver compiles against, so it +//! must not drag in the OpenHuman host; and the generic subsystem vocabulary +//! (`Driver`, `DriverClass`, `SubsystemRegistry`, the policy `Guard`) belongs +//! kernel-side, where inference and channels can share it without importing a +//! *memory* crate. +//! +//! The bridge is the host's memory adapter, which implements the host `Driver` +//! for an `Arc` and converts [`MemoryHealth`] into the +//! kernel's `DriverHealth`. That conversion is trivial by construction — see +//! [`crate::openhuman::memory::api::health`]. +//! +//! Driver **class** (embedded / external / null) is deliberately absent from +//! this trait. Class is a fact about how the host bound a driver, recorded in +//! host configuration; a driver self-reporting it would let a misconfigured +//! external backend claim to be embedded and skip the egress and trust checks +//! that class gates. +//! +//! ## The accessor form, and why not `Any` +//! +//! The kernel binds `Arc` and needs per-family access. Two +//! designs were available: downcast through [`std::any::Any`], or one +//! `Option`-returning accessor per optional family. The accessors win: +//! +//! - **No unchecked downcast.** `Any` would require the caller to name a +//! concrete driver type, which defeats the point of binding behind a trait +//! object, or to register type ids, which is the same table with worse +//! ergonomics. +//! - **The capability set and the reachable surface stay provably in sync.** +//! [`crate::openhuman::memory::api::provider::audit_provider`] compares [`MemoryProvider::capabilities`] +//! against what the accessors actually return, so "advertised but not +//! implemented" is a detectable, testable mistake instead of a runtime +//! surprise on the first call. +//! - **[`MemoryProvider::provides`] is an exhaustive `match`** over +//! [`Capability`], so adding a family without wiring an accessor fails to +//! compile. +//! +//! The three mandatory families are supertraits rather than accessors, so they +//! are callable directly on the trait object and cannot be absent. +//! +//! ## Object safety +//! +//! Every method here and in every family trait is object-safe: no generic +//! parameters, no `Self` in return position, no associated constants. The +//! `#[async_trait]` attribute rewrites the `async fn`s into boxed futures, +//! which is what makes them dyn-compatible at all. + +use async_trait::async_trait; + +use crate::openhuman::memory::api::capabilities::{Capabilities, Capability}; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::health::MemoryHealth; +use crate::openhuman::memory::api::provider::content::{MemoryDocuments, MemoryIngest, MemoryTree}; +use crate::openhuman::memory::api::provider::knowledge::{MemoryDiff, MemoryEntities, MemoryGraph}; +use crate::openhuman::memory::api::provider::mandatory::{ + MemoryCore, MemoryPortability, MemoryRecall, +}; +use crate::openhuman::memory::api::provider::records::{ + MemoryGoals, MemoryMaintenance, MemorySourceSink, MemoryToolMemory, +}; + +/// A bound memory driver. +/// +/// Implementors must also implement the three mandatory families +/// ([`MemoryCore`], [`MemoryRecall`], [`MemoryPortability`]) — they are +/// supertraits, so a driver missing any of them cannot be constructed as a +/// provider at all. +/// +/// The ten optional families are reached through the `as_*` accessors below. +/// Each defaults to `None`, so a minimal driver implements only what it +/// supports and inherits correct absence for everything else. +#[async_trait] +pub trait MemoryProvider: MemoryCore + MemoryRecall + MemoryPortability + 'static { + /// Stable identifier for this driver (`tinycortex`, `supermemory`, `null`). + /// + /// Appears in status output, log lines, tracing spans, and audit events, so + /// it must be stable across restarts and must not embed a URL, a token, or + /// anything else user- or deployment-specific. + fn driver_id(&self) -> &str; + + /// The families this driver implements. + /// + /// Asked **once** at bind time and cached: the kernel filters RPC + /// registration and agent-tool assembly from the cached answer, so a set + /// that changes after binding will not be noticed. A driver whose surface + /// genuinely varies must report the union and answer + /// [`MemoryError::Unsupported`] for the gaps. + /// + /// Must be honest: every advertised family must be reachable through its + /// accessor. [`crate::openhuman::memory::api::provider::audit_provider`] checks exactly that. + fn capabilities(&self) -> Capabilities; + + /// Current liveness, as the driver reports it. + /// + /// Called on bind and on demand for status output. Implementations should + /// be cheap and must not block indefinitely — a health probe that hangs is + /// indistinguishable from a subsystem that is down, but takes a timeout to + /// find out. + async fn health(&self) -> MemoryHealth; + + /// Release resources ahead of process exit or a rebind. + /// + /// Defaults to a successful no-op, because most drivers have nothing to + /// release; a driver holding a connection pool or a background task should + /// override it. The host's adapter forwards its `Driver::shutdown` here. + /// + /// Must be idempotent: a rebind followed by process exit calls it twice. + /// + /// # Errors + /// + /// Backend failures during teardown. The caller logs and continues — + /// shutdown failure never blocks exit. + async fn shutdown(&self) -> Result<(), MemoryError> { + Ok(()) + } + + /// Bulk ingestion, when advertised. + fn as_ingest(&self) -> Option<&dyn MemoryIngest> { + None + } + + /// The namespace-document tier, when advertised. + fn as_documents(&self) -> Option<&dyn MemoryDocuments> { + None + } + + /// The summary tree, when advertised. + fn as_tree(&self) -> Option<&dyn MemoryTree> { + None + } + + /// The entity index, when advertised. + fn as_entities(&self) -> Option<&dyn MemoryEntities> { + None + } + + /// The key/value and relation graph, when advertised. + fn as_graph(&self) -> Option<&dyn MemoryGraph> { + None + } + + /// Snapshot and change tracking, when advertised. + fn as_diff(&self) -> Option<&dyn MemoryDiff> { + None + } + + /// The long-term goals document, when advertised. + fn as_goals(&self) -> Option<&dyn MemoryGoals> { + None + } + + /// Per-tool learned rules, when advertised. + fn as_tool_memory(&self) -> Option<&dyn MemoryToolMemory> { + None + } + + /// The host-sync write seam, when advertised. + fn as_sources(&self) -> Option<&dyn MemorySourceSink> { + None + } + + /// Scheduler-driven upkeep, when advertised. + fn as_maintenance(&self) -> Option<&dyn MemoryMaintenance> { + None + } + + /// Whether `capability` is actually **reachable** on this driver. + /// + /// This is the implementation-side truth, as opposed to + /// [`Self::capabilities`], which is the advertised claim. The two should + /// agree; [`crate::openhuman::memory::api::provider::audit_provider`] is where they are compared. + /// + /// The mandatory three are always `true` because they are supertraits. The + /// remaining ten delegate to their accessor. + /// + /// The `match` is deliberately exhaustive: [`Capability`] is not + /// `#[non_exhaustive]`, so adding a family without adding an accessor and + /// an arm here is a compile error rather than a silent `false`. + fn provides(&self, capability: Capability) -> bool { + match capability { + Capability::Core | Capability::Recall | Capability::Portability => true, + Capability::Ingest => self.as_ingest().is_some(), + Capability::Documents => self.as_documents().is_some(), + Capability::Tree => self.as_tree().is_some(), + Capability::Entities => self.as_entities().is_some(), + Capability::Graph => self.as_graph().is_some(), + Capability::Diff => self.as_diff().is_some(), + Capability::Goals => self.as_goals().is_some(), + Capability::ToolMemory => self.as_tool_memory().is_some(), + Capability::Sources => self.as_sources().is_some(), + Capability::Maintenance => self.as_maintenance().is_some(), + } + } +} diff --git a/src/openhuman/memory/api/provider/knowledge.rs b/src/openhuman/memory/api/provider/knowledge.rs new file mode 100644 index 0000000000..2a49a3a9a8 --- /dev/null +++ b/src/openhuman/memory/api/provider/knowledge.rs @@ -0,0 +1,179 @@ +//! Optional families that expose *derived structure* over stored memory: +//! [`MemoryEntities`], [`MemoryGraph`], and [`MemoryDiff`]. +//! +//! Each is independently optional. A driver may have a key/value graph but no +//! entity index, or track source snapshots without either. The kernel filters +//! RPC registration and agent-tool assembly per family, so an absent family is +//! invisible rather than present-and-failing. +//! +//! As in [`crate::openhuman::memory::api::provider::content`], no configuration crosses this boundary: +//! extraction models, hotness decay curves, and snapshot retention are driver +//! concerns and appear in none of these signatures. + +use async_trait::async_trait; + +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::provider::types::{DiffReport, EntityHit, SnapshotRef}; +use crate::openhuman::memory::api::types::{GraphRelationRecord, MemoryKvRecord}; + +/// The entity index: who and what the stored memory is about. +#[async_trait] +pub trait MemoryEntities: Send + Sync { + /// List entities in a namespace, ranked by hotness when `query` is `None` + /// and by match quality otherwise. + /// + /// # Errors + /// + /// Backend failures only; an unknown namespace yields an empty vector. + async fn entities( + &self, + namespace: &str, + query: Option<&str>, + limit: usize, + ) -> Result, MemoryError>; + + /// Edges incident to one entity, most relevant first. + /// + /// Returns [`GraphRelationRecord`] — the same shape [`MemoryGraph`] uses — + /// so a caller that has both families does not have to reconcile two edge + /// representations. + /// + /// # Errors + /// + /// Backend failures only; an unknown `entity_id` yields an empty vector + /// rather than [`MemoryError::NotFound`], because "no edges" and "no such + /// entity" are the same answer to this question. + async fn entity_edges( + &self, + namespace: &str, + entity_id: &str, + limit: usize, + ) -> Result, MemoryError>; + + /// Record that these entities were just observed, updating hotness. + /// + /// Separate from the read path because hotness is a *write* the host + /// triggers at known moments (a turn referenced these entities), not + /// something a driver should infer from being queried — otherwise merely + /// browsing the index would reshape ranking. + /// + /// # Errors + /// + /// Backend failures only. Unknown ids are ignored, not rejected. + async fn touch_entities( + &self, + namespace: &str, + entity_ids: &[String], + ) -> Result<(), MemoryError>; +} + +/// The key/value and relation graph tier. +/// +/// `namespace` is `Option<&str>` throughout: `None` addresses the global, +/// namespace-less slice, matching the storage shape of +/// [`MemoryKvRecord::namespace`] and [`GraphRelationRecord::namespace`]. +#[async_trait] +pub trait MemoryGraph: Send + Sync { + /// Read one key/value record. + /// + /// # Errors + /// + /// A missing key is `Ok(None)`; `Err` is reserved for backend failures. + async fn kv_get( + &self, + namespace: Option<&str>, + key: &str, + ) -> Result, MemoryError>; + + /// Upsert one key/value record. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a rejected key, otherwise backend failures. + async fn kv_put( + &self, + namespace: Option<&str>, + key: &str, + value: serde_json::Value, + ) -> Result<(), MemoryError>; + + /// Delete one key/value record, reporting whether it existed. + async fn kv_delete(&self, namespace: Option<&str>, key: &str) -> Result; + + /// List key/value records, optionally restricted to a key prefix. + /// + /// # Errors + /// + /// Backend failures only. + async fn kv_list( + &self, + namespace: Option<&str>, + prefix: Option<&str>, + limit: usize, + ) -> Result, MemoryError>; + + /// Query relations, narrowing by subject and/or predicate. + /// + /// Both filters are `None`-able so one method covers "everything about this + /// subject", "every edge of this type", and "the whole slice", instead of + /// three near-identical methods. + /// + /// # Errors + /// + /// Backend failures only. + async fn relations( + &self, + namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + limit: usize, + ) -> Result, MemoryError>; + + /// Upsert one relation, keyed by `(namespace, subject, predicate, object)`. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a malformed edge, otherwise backend + /// failures. + async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError>; +} + +/// Snapshot capture and change computation over synced sources. +#[async_trait] +pub trait MemoryDiff: Send + Sync { + /// Capture a snapshot of one source's current items. + /// + /// # Errors + /// + /// [`MemoryError::NotFound`] for an unknown `source_id`, otherwise backend + /// failures. + async fn capture_snapshot(&self, source_id: &str) -> Result; + + /// List snapshots for one source, newest first. + /// + /// # Errors + /// + /// Backend failures only; an unknown `source_id` yields an empty vector. + async fn snapshots( + &self, + source_id: &str, + limit: usize, + ) -> Result, MemoryError>; + + /// Compute the change set between two snapshots of one source. + /// + /// `from` is `Option<&str>` so the first-ever diff — where there is no + /// baseline and every item is an addition — is expressible without a + /// separate method or a sentinel id. + /// + /// # Errors + /// + /// [`MemoryError::NotFound`] when either snapshot id is unknown, otherwise + /// backend failures. + async fn diff( + &self, + source_id: &str, + from: Option<&str>, + to: &str, + ) -> Result; +} diff --git a/src/openhuman/memory/api/provider/mandatory.rs b/src/openhuman/memory/api/provider/mandatory.rs new file mode 100644 index 0000000000..4699bcea2a --- /dev/null +++ b/src/openhuman/memory/api/provider/mandatory.rs @@ -0,0 +1,188 @@ +//! The three mandatory capability families: [`MemoryCore`], [`MemoryRecall`], +//! and [`MemoryPortability`]. +//! +//! These are supertraits of [`crate::openhuman::memory::api::provider::MemoryProvider`], which is what +//! makes "mandatory" a *compile-time* fact rather than a runtime check: a type +//! that does not implement all three cannot be a provider at all, so there is +//! no way to bind a driver that is missing them. +//! +//! The other ten families are reached through `Option`-returning accessors on +//! the provider, so their absence is representable and their presence is not +//! assumed. See [`crate::openhuman::memory::api::provider::MemoryProvider`] for that half. +//! +//! ## Why every method returns [`MemoryError`] and not `anyhow::Error` +//! +//! The transport adapter must be able to turn a `501` from an out-of-process +//! driver into [`MemoryError::Unsupported`], and the kernel must be able to +//! tell "this driver cannot do that" apart from "this driver failed". An +//! `anyhow::Error` erases exactly that distinction. The engine's own +//! [`crate::openhuman::memory::api::traits::Memory`] trait keeps `anyhow::Result` — it is an internal +//! storage abstraction with existing implementors, not the driver contract. + +use async_trait::async_trait; + +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::provider::types::{ + ExportPage, ExportRecord, ImportOutcome, SourceScope, +}; +use crate::openhuman::memory::api::recall::OwnedRecallOpts; +use crate::openhuman::memory::api::types::{ + MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, +}; + +/// Store, read, and delete individual memory entries. **Mandatory.** +/// +/// This is the smallest surface that still makes something a memory backend: +/// without it there is nothing to recall from and nothing to export. +#[async_trait] +pub trait MemoryCore: Send + Sync { + /// Upsert an entry, keyed by `(namespace, key)`. + /// + /// ## Taint is an argument, never a decision + /// + /// Unlike the engine's [`crate::openhuman::memory::api::traits::Memory`], which has a `store` and a + /// separate `store_with_taint` whose default implementation silently drops + /// the taint, the contract has **one** store and it always takes a + /// [`MemoryTaint`]. Provenance is stamped by the host policy guard before + /// the call; a driver that could default it would be able to launder + /// externally-sourced content into internal-trust content, which is the + /// single failure mode the guard exists to prevent. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for caller input the driver rejects, + /// [`MemoryError::Io`] or [`MemoryError::Other`] for backend failures. + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> Result<(), MemoryError>; + + /// Fetch the entry for an exact `(namespace, key)`. + /// + /// # Errors + /// + /// A missing entry is `Ok(None)`, never an error; `Err` is reserved for + /// backend failures. + async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError>; + + /// Delete the entry for `(namespace, key)`, reporting whether it existed. + /// + /// Idempotent: forgetting an absent key is `Ok(false)`, so callers may call + /// it unconditionally. + /// + /// # Errors + /// + /// Backend failures only. + async fn forget(&self, namespace: &str, key: &str) -> Result; + + /// List entries, narrowing by namespace, category, and session. + /// + /// Each `Some` filter narrows the result; all `None` lists everything the + /// driver holds. An empty result is `Ok(vec![])`. + /// + /// # Errors + /// + /// Backend failures only. + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> Result, MemoryError>; + + /// Enumerate namespaces with their aggregate counts, for discovery. + /// + /// # Errors + /// + /// Backend failures only. + async fn namespaces(&self) -> Result, MemoryError>; +} + +/// Ranked retrieval. **Mandatory.** +#[async_trait] +pub trait MemoryRecall: Send + Sync { + /// Return up to `limit` entries relevant to `query`, most relevant first. + /// + /// `opts` is the **owned** [`OwnedRecallOpts`], never the borrowed + /// `RecallOpts<'a>`: a lifetime parameter cannot travel through an + /// object-safe `#[async_trait]` method, and the borrowed form derives no + /// serde impls so it could never be a request body. An embedded driver + /// converts to the borrowed form at its own boundary, which is zero-copy. + /// + /// `scope` is the per-turn source allowlist and is a **query predicate the + /// driver must apply internally** — see [`SourceScope`] for why applying it + /// after the fact is wrong. `None` means unrestricted. + /// + /// An empty or non-matching `query` yields `Ok(vec![])`, not an error. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a malformed filter, otherwise backend + /// failures. + async fn recall( + &self, + query: &str, + limit: usize, + opts: &OwnedRecallOpts, + scope: Option<&SourceScope>, + ) -> Result, MemoryError>; +} + +/// Export and import the whole store. **Mandatory.** +/// +/// Mandatory because binding a memory backend without it is a one-way door: a +/// user who cannot export cannot leave. It is the capability that makes every +/// other binding reversible, which is also why the `mirror` migration driver is +/// expressible at all. +#[async_trait] +pub trait MemoryPortability: Send + Sync { + /// Read one page of the export, continuing from `cursor`. + /// + /// Pass `None` to start. The export is complete when the returned + /// [`ExportPage::next_cursor`] is `None` — an empty `records` vector is + /// **not** a terminator, because a driver may legitimately return an empty + /// page while skipping a range. + /// + /// `limit` is a request, not a guarantee; a driver may return fewer. + /// + /// ## Why pages and not a stream + /// + /// A `Stream` return type would either make the trait non-object-safe or + /// drag an async runtime into a crate that deliberately has none. Paging + /// keeps both properties and still bounds memory, with the caller choosing + /// the bound. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a cursor this driver did not issue, + /// otherwise backend failures. + async fn export_page( + &self, + cursor: Option<&str>, + limit: usize, + ) -> Result; + + /// Write a batch of previously-exported records. + /// + /// Records carry their own [`crate::openhuman::memory::api::types::MemoryTaint`]; an importing + /// driver must persist what it is given and must not re-stamp provenance. + /// + /// Partial success is normal and is reported in [`ImportOutcome`] rather + /// than as an error: a migration should not abort a million-record restore + /// because one record was malformed. + /// + /// # Errors + /// + /// Reserved for failures that make the whole batch meaningless (backend + /// unavailable, transaction aborted). Per-record rejection belongs in + /// [`ImportOutcome::failed`]. + async fn import_records( + &self, + records: Vec, + ) -> Result; +} diff --git a/src/openhuman/memory/api/provider/mod.rs b/src/openhuman/memory/api/provider/mod.rs new file mode 100644 index 0000000000..8026d1fb9b --- /dev/null +++ b/src/openhuman/memory/api/provider/mod.rs @@ -0,0 +1,73 @@ +//! The memory driver contract: [`MemoryProvider`] plus the thirteen capability +//! family traits a driver may implement. +//! +//! ## Shape +//! +//! ```text +//! MemoryProvider ── identity, capabilities, health, shutdown +//! : MemoryCore (mandatory — supertrait, always callable) +//! : MemoryRecall (mandatory — supertrait, always callable) +//! : MemoryPortability (mandatory — supertrait, always callable) +//! ├─ as_ingest() -> Option<&dyn MemoryIngest> +//! ├─ as_documents() -> Option<&dyn MemoryDocuments> +//! ├─ as_tree() -> Option<&dyn MemoryTree> +//! ├─ as_entities() -> Option<&dyn MemoryEntities> +//! ├─ as_graph() -> Option<&dyn MemoryGraph> +//! ├─ as_diff() -> Option<&dyn MemoryDiff> +//! ├─ as_goals() -> Option<&dyn MemoryGoals> +//! ├─ as_tool_memory() -> Option<&dyn MemoryToolMemory> +//! ├─ as_sources() -> Option<&dyn MemorySourceSink> +//! └─ as_maintenance() -> Option<&dyn MemoryMaintenance> +//! ``` +//! +//! The mandatory three are supertraits, so "mandatory" is enforced by the type +//! system rather than by a runtime check. The optional ten are accessors that +//! default to `None`, so absence is the default and presence is opt-in. +//! +//! ## Rules that bind every family +//! +//! 1. **Typed errors, always.** Every method returns +//! `Result<_, MemoryError>`. The transport adapter maps an out-of-process +//! `501` onto [`crate::openhuman::memory::api::error::MemoryError::Unsupported`], and the kernel +//! distinguishes "cannot" from "failed". `anyhow::Error` would erase that. +//! 2. **No configuration crosses the boundary.** Not one signature names a +//! config type. A driver holds its own configuration; the contract passes +//! domain arguments only. +//! 3. **No host types.** Nothing here names an OpenHuman type, so a +//! third-party driver depends on this crate alone. +//! 4. **The driver never assigns provenance.** [`crate::openhuman::memory::api::types::MemoryTaint`] is +//! an argument on every write path and a preserved field on every import. +//! 5. **The host owns the loop.** Sealing, cascading, maintenance, and source +//! sync are all "run one step when asked"; no driver installs a background +//! task or hooks the agent turn. +//! 6. **Object safety throughout.** No generics, no `Self` returns, no +//! associated constants — every family is usable as `&dyn`. +//! +//! ## Reference implementation +//! +//! [`crate::openhuman::memory::api::null::NullMemoryProvider`] implements all thirteen families: +//! `/dev/null` semantics for the mandatory three, and +//! [`crate::openhuman::memory::api::error::MemoryError::Unsupported`] for the other ten, which it does +//! not advertise. It is what a compiled-out or unconfigured memory subsystem +//! binds to, and it doubles as the proof that the mandatory set is +//! implementable without a storage engine. + +pub mod audit; +pub mod content; +pub mod driver; +pub mod knowledge; +pub mod mandatory; +pub mod records; +pub mod types; + +pub use audit::{audit_provider, CapabilityAudit}; +pub use content::{MemoryDocuments, MemoryIngest, MemoryTree}; +pub use driver::MemoryProvider; +pub use knowledge::{MemoryDiff, MemoryEntities, MemoryGraph}; +pub use mandatory::{MemoryCore, MemoryPortability, MemoryRecall}; +pub use records::{MemoryGoals, MemoryMaintenance, MemorySourceSink, MemoryToolMemory}; +pub use types::{ + ChangeKind, DiffReport, EntityHit, EntityRef, ExportPage, ExportRecord, ImportOutcome, + IngestItem, IngestOutcome, MaintenanceReport, SnapshotRef, SourceChange, SourceItem, + SourceScope, +}; diff --git a/src/openhuman/memory/api/provider/records.rs b/src/openhuman/memory/api/provider/records.rs new file mode 100644 index 0000000000..5b8ee13a67 --- /dev/null +++ b/src/openhuman/memory/api/provider/records.rs @@ -0,0 +1,169 @@ +//! The remaining optional families: [`MemoryGoals`], [`MemoryToolMemory`], +//! [`MemorySourceSink`], and [`MemoryMaintenance`]. +//! +//! Goals and tool memory are small curated record sets the agent reads on +//! nearly every turn. The source sink is the seam the host's sync machinery +//! writes through. Maintenance is the seam the host's scheduler drives. +//! +//! ## The host keeps the loop; the driver runs one step +//! +//! [`MemorySourceSink`] receives already-fetched items — the host owns +//! credentials, OAuth, rate limits, and the schedule. [`MemoryMaintenance`] +//! exposes four operations the host's existing scheduler calls; no driver +//! installs a background task of its own. Both follow the same rule as the +//! engine's `queue::run_once`, and both are why a driver never needs to see +//! configuration or a keychain. + +use async_trait::async_trait; + +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::goals::GoalsDoc; +use crate::openhuman::memory::api::provider::types::{ + IngestOutcome, MaintenanceReport, SourceItem, +}; +use crate::openhuman::memory::api::tool_memory::ToolMemoryRule; +use crate::openhuman::memory::api::types::MemoryTaint; + +/// The agent's long-term goals document. +#[async_trait] +pub trait MemoryGoals: Send + Sync { + /// Read the current goals document. + /// + /// A driver with no goals yet returns an empty [`GoalsDoc`], not + /// [`MemoryError::NotFound`] — "no goals" is a valid state, not a missing + /// record. + /// + /// # Errors + /// + /// Backend failures only. + async fn goals(&self) -> Result; + + /// Replace the goals document wholesale. + /// + /// Whole-document replacement rather than per-item add/edit/delete because + /// the validating mutation surface (PII and secret predicates) is **host** + /// policy: the host parses, validates, mutates, and hands back the result. + /// Exposing per-item mutation here would put that policy behind a trait a + /// third-party driver implements, where it could be skipped. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a document the driver refuses (e.g. over + /// its own item cap), otherwise backend failures. + async fn set_goals(&self, goals: GoalsDoc) -> Result<(), MemoryError>; +} + +/// Per-tool learned rules — durable guidance attached to a specific tool. +#[async_trait] +pub trait MemoryToolMemory: Send + Sync { + /// Rules for one tool, highest priority first. + /// + /// # Errors + /// + /// Backend failures only; a tool with no rules yields an empty vector. + async fn tool_rules(&self, tool_name: &str) -> Result, MemoryError>; + + /// Upsert one rule, keyed by [`ToolMemoryRule::id`]. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a malformed rule, otherwise backend + /// failures. + async fn put_tool_rule(&self, rule: ToolMemoryRule) -> Result<(), MemoryError>; + + /// Delete one rule, reporting whether it existed. + /// + /// Idempotent, like [`crate::openhuman::memory::api::provider::MemoryCore::forget`]. + /// + /// # Errors + /// + /// Backend failures only. + async fn delete_tool_rule(&self, tool_name: &str, rule_id: &str) -> Result; +} + +/// The write seam for host-driven source sync. +#[async_trait] +pub trait MemorySourceSink: Send + Sync { + /// Accept a batch of items the host fetched from one logical source. + /// + /// `taint` applies to the whole batch and is stamped by the host. Sync + /// paths ingesting third-party content pass + /// [`MemoryTaint::ExternalSync`]; the driver persists what it is given and + /// never assigns provenance itself. + /// + /// `source_kind` is a wire string (`folder`, `composio`, …) rather than an + /// enum because the set of source kinds is owned by the host's sync + /// machinery and grows without a contract change. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a rejected batch, otherwise backend + /// failures. Per-item outcomes are counted in [`IngestOutcome`]. + async fn accept_source_items( + &self, + source_id: &str, + source_kind: &str, + items: Vec, + taint: MemoryTaint, + ) -> Result; + + /// Drop everything the driver holds for one logical source, returning how + /// many units were removed. + /// + /// This is the disconnect path: when a user removes a source, its content + /// must leave memory. Idempotent — an unknown `source_id` returns `Ok(0)`. + /// + /// # Errors + /// + /// Backend failures only. + async fn forget_source(&self, source_id: &str) -> Result; +} + +/// Periodic upkeep the host's scheduler drives. +/// +/// All four operations must be safe to call repeatedly and safe to interrupt: +/// the scheduler may invoke them on a timer, and a desktop process can exit at +/// any point. A driver that cannot bound the work should do a slice per call +/// and report progress in [`MaintenanceReport`]. +#[async_trait] +pub trait MemoryMaintenance: Send + Sync { + /// Recompute embeddings for content whose embedding is missing or stale. + /// + /// # Errors + /// + /// Backend failures, or [`MemoryError::BudgetExceeded`] when an embedding + /// budget is exhausted mid-run. + async fn reembed(&self) -> Result; + + /// Reclaim space: vacuum indexes, drop tombstones, prune dead references. + /// + /// # Errors + /// + /// Backend failures only. + async fn compact(&self) -> Result; + + /// Merge and summarise accumulated memory — the "dream" pass. + /// + /// The embedded driver maps this onto its seal/cascade/reembed cycle; an + /// external driver maps it onto whatever it calls the same idea. The + /// contract deliberately does not specify the mechanism, only that it is + /// the operation a scheduler runs when the system is idle. + /// + /// # Errors + /// + /// Backend failures only. + async fn consolidate(&self) -> Result; + + /// Read-only integrity check. + /// + /// Reports findings in [`MaintenanceReport::findings`] and must change + /// nothing — [`MaintenanceReport::changed`] is always `0`. A driver that + /// repairs as it inspects should expose that as [`Self::compact`] instead, + /// so an operator can diagnose without mutating. + /// + /// # Errors + /// + /// Backend failures only. A *finding* is not an error: a store with + /// problems still returns `Ok` with the problems listed. + async fn doctor(&self) -> Result; +} diff --git a/src/openhuman/memory/api/provider/types.rs b/src/openhuman/memory/api/provider/types.rs new file mode 100644 index 0000000000..830d0af3c3 --- /dev/null +++ b/src/openhuman/memory/api/provider/types.rs @@ -0,0 +1,393 @@ +//! Value types that exist only because the *driver contract* needs them. +//! +//! Everything here is inert data: serde-derived, dependency-light, and free of +//! any engine or host type. They are separated from [`crate::openhuman::memory::api::types`] because +//! that module carries the historical engine value types (which the engine +//! crate aliases back into `crate::openhuman::memory::engine::types`), whereas these are new +//! shapes introduced by the provider contract itself. +//! +//! ## Why these types and not the engine's +//! +//! Several families the contract exposes (diff, entities, sources, +//! maintenance) have richer types inside the `tinycortex` engine — for example +//! `memory::diff::types::DiffResult`. Those types are *implementation* shapes: +//! they carry git commit SHAs, ledger paths, and engine-specific enums. A +//! third-party driver cannot produce them and must not be required to. +//! +//! So the contract defines the narrower shape a *caller* actually needs, with +//! wire strings deliberately identical to the engine's where they overlap +//! (`added`/`removed`/`modified`), so the embedded driver's conversion is a +//! field-for-field map rather than a translation. +//! +//! ## What is deliberately absent +//! +//! No type here names a configuration struct. `MemoryConfig` stayed engine-side +//! in the M0 carve-out and stays there: a driver holds its own configuration +//! and the contract passes only domain arguments. If a future method cannot be +//! expressed without configuration, that is a signal the family was designed +//! wrong, not that the contract should widen. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::openhuman::memory::api::chunks::{DataSource, SourceRef}; +use crate::openhuman::memory::api::types::MemoryTaint; + +/// A per-turn allowlist of memory sources, passed **into** the driver as a +/// query predicate. +/// +/// ## Why this is a parameter and not a post-filter +/// +/// The host computes a per-turn source allowlist from product policy. If that +/// allowlist were applied after the driver returned rows, a `limit` would be +/// consumed by rows the caller is not allowed to see — so a scoped query could +/// return fewer results than it should, or none at all, purely as an artefact +/// of filtering order. Worse, an out-of-process driver would have already been +/// handed a query it should never have answered in full. +/// +/// The predicate therefore travels with the call. `None` means unrestricted; +/// `Some(scope)` means the driver must apply it *inside* its query. +/// +/// ## Matching rule (fail-closed) +/// +/// [`SourceScope::allows_source_id`] encodes the embedded engine's SQL +/// semantics verbatim: a source-attributed id is in scope when it either equals +/// an allowed id outright, or begins with `mem_src:{allowed}:`. An **empty** +/// allow list therefore matches nothing — a scope that lists no sources denies +/// all source-attributed content rather than waving it through. +/// +/// Content that is not attributed to a memory source at all (no +/// `memory_sources` provenance) is outside this predicate's remit; the driver +/// decides that, exactly as the engine's SQL does today. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceScope { + /// Allowed memory-source identifiers. Empty denies all source-attributed + /// content. + pub allow: Vec, +} + +impl SourceScope { + /// Builds a scope from any iterator of source identifiers. + pub fn new(allow: impl IntoIterator>) -> Self { + Self { + allow: allow.into_iter().map(Into::into).collect(), + } + } + + /// Whether this scope lists no sources — in which case it denies all + /// source-attributed content. See the type docs for why that is the + /// fail-closed reading and not "unrestricted". + pub fn is_empty(&self) -> bool { + self.allow.is_empty() + } + + /// Whether `source_id` is in scope, using the engine's equality-or-prefix + /// rule. + /// + /// ``` + /// use openhuman_core::openhuman::memory::api::provider::types::SourceScope; + /// + /// let scope = SourceScope::new(["src-abc"]); + /// assert!(scope.allows_source_id("src-abc")); + /// assert!(scope.allows_source_id("mem_src:src-abc:item-1")); + /// assert!(!scope.allows_source_id("src-xyz")); + /// + /// // An empty scope denies everything. + /// assert!(!SourceScope::default().allows_source_id("src-abc")); + /// ``` + pub fn allows_source_id(&self, source_id: &str) -> bool { + self.allow.iter().any(|allowed| { + source_id == allowed || source_id.starts_with(&format!("mem_src:{allowed}:")) + }) + } +} + +/// One unit of content handed to [`crate::openhuman::memory::api::provider::MemoryIngest`]. +/// +/// The driver owns chunking, embedding, and persistence — this type carries +/// only what the driver cannot know: where the content came from, when, who it +/// belongs to, and how far it may be trusted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct IngestItem { + /// Target namespace; `None` means the driver's default namespace. + #[serde(default)] + pub namespace: Option, + /// Concrete upstream provider the content came from. + pub source: DataSource, + /// Stable logical id for the ingestion group (channel id, thread id, doc + /// id). This is the dedupe key, not a display value. + pub source_id: String, + /// Account or user the content belongs to; empty for anonymous/system + /// sources. + #[serde(default)] + pub owner: String, + /// Opaque pointer back to the raw source record, for citation and + /// drill-down. + #[serde(default)] + pub source_ref: Option, + /// The content itself, already decoded to text. + pub content: String, + /// MIME type of [`Self::content`] when the caller knows it. + #[serde(default)] + pub mime: Option, + /// Event time used for ordering and tree placement; the driver substitutes + /// ingest time when absent. + #[serde(default)] + pub timestamp: Option>, + /// Labels carried through from the source. Ingest does not interpret them. + #[serde(default)] + pub tags: Vec, + /// Provenance taint. The **host** stamps this; a driver must persist what it + /// is given and must never assign or upgrade it. + #[serde(default)] + pub taint: MemoryTaint, + /// Overrides `source_id` for on-disk path grouping only; `source_id` + /// remains the dedupe key. + #[serde(default)] + pub path_scope: Option, +} + +/// What an ingest call actually persisted. +/// +/// Counts rather than content, so the caller can report progress and detect a +/// silently-dropping driver without holding the written material in memory. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct IngestOutcome { + /// Units the driver newly persisted. + pub written: u32, + /// Units the driver recognised as already present and skipped. + pub skipped: u32, + /// Driver-assigned ids for the written units, when the driver exposes them. + /// May be empty even when [`Self::written`] is non-zero — an external + /// backend is not obliged to surface its internal ids. + #[serde(default)] + pub ids: Vec, +} + +/// One line of the portability stream. +/// +/// Export and import are defined over records rather than bytes so the contract +/// stays free of an async runtime and of any streaming abstraction: the host +/// adapter turns a page of records into NDJSON (and back) at the transport +/// boundary. +/// +/// [`Self::kind`] is a driver-defined string rather than an enum. A backend has +/// record kinds this crate has never heard of, and a migration between two +/// backends must round-trip them untouched rather than drop what it cannot +/// classify. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ExportRecord { + /// Driver-defined record kind (e.g. `entry`, `document`, `chunk`). + pub kind: String, + /// Driver-assigned id, unique within [`Self::kind`]. + pub id: String, + /// Owning namespace, when the record has one. + #[serde(default)] + pub namespace: Option, + /// Provenance taint of the record's content. Preserved across + /// export → import; an importing driver must not re-stamp it. + #[serde(default)] + pub taint: MemoryTaint, + /// The record body, in the exporting driver's own shape. + pub payload: serde_json::Value, +} + +/// One page of an export, plus the cursor that continues it. +/// +/// Paging (rather than a stream) keeps [`crate::openhuman::memory::api::provider::MemoryPortability`] +/// object-safe and runtime-agnostic while still bounding memory: the caller +/// decides the page size and drives the loop. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ExportPage { + /// Records in this page. May be empty on the final page. + pub records: Vec, + /// Opaque cursor to pass to the next call. `None` means the export is + /// complete — this, not an empty [`Self::records`], is the terminator. + #[serde(default)] + pub next_cursor: Option, +} + +/// What an import call actually accepted. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImportOutcome { + /// Records written. + pub imported: u32, + /// Records recognised as already present and skipped. + pub skipped: u32, + /// Records rejected. A non-zero value with an empty [`Self::errors`] is a + /// driver bug: a rejection the operator cannot diagnose. + pub failed: u32, + /// Operator-facing reasons for the failures, bounded by the driver. Must + /// not contain record content or credentials — this is logged. + #[serde(default)] + pub errors: Vec, +} + +/// Identity of an entity in the driver's index. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EntityRef { + /// Canonical, driver-stable entity id. + pub id: String, + /// Entity kind as a wire string (`person`, `organization`, `topic`, …). + /// A string rather than an enum because the taxonomy is the driver's, and a + /// kind this build does not recognise must still round-trip. + pub kind: String, + /// Display name. + pub name: String, +} + +/// An entity together with its recency/frequency signals. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct EntityHit { + /// The entity itself. + pub entity: EntityRef, + /// Driver-computed hotness, higher is hotter. Not normalised across + /// drivers — compare within one driver's results only. + pub hotness: f64, + /// Number of times the entity was observed. + pub mentions: u32, +} + +/// Identity of a captured snapshot. +/// +/// The engine's own snapshot type additionally carries the git commit SHA and +/// ledger trailers that back it; those are implementation, so the contract +/// exposes only the identity and the counts a caller can act on. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SnapshotRef { + /// Driver-stable snapshot id. + pub id: String, + /// Logical source this snapshot covers. + pub source_id: String, + /// Human-readable source label at capture time. + #[serde(default)] + pub label: String, + /// Number of items materialised into the snapshot. + pub item_count: u32, + /// Capture time in milliseconds since the Unix epoch. + pub taken_at_ms: i64, +} + +/// What happened to one item between two snapshots. +/// +/// Wire strings are identical to the engine's `memory::diff::types::ChangeKind` +/// so the embedded adapter maps rather than translates. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChangeKind { + /// Present in the later snapshot only. + Added, + /// Present in the earlier snapshot only. + Removed, + /// Present in both, with differing content. + Modified, +} + +impl ChangeKind { + /// Stable wire string. + pub fn as_str(self) -> &'static str { + match self { + Self::Added => "added", + Self::Removed => "removed", + Self::Modified => "modified", + } + } +} + +/// A single item-level change inside a [`DiffReport`]. +/// +/// Item identity is the item id, never the title, so a rename reports as a +/// removal plus an addition rather than a modification. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceChange { + /// Stable item id. + pub item_id: String, + /// Display title, or the id when the driver has no better label. + #[serde(default)] + pub title: String, + /// What kind of change occurred. + pub kind: ChangeKind, + /// Content hash on the earlier side; absent for an addition. + #[serde(default)] + pub old_content_hash: Option, + /// Content hash on the later side; absent for a removal. + #[serde(default)] + pub new_content_hash: Option, +} + +/// The result of diffing one source between two snapshots. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DiffReport { + /// Source this diff covers. + pub source_id: String, + /// Baseline snapshot id; `None` for a first-ever diff, where everything is + /// an addition. + #[serde(default)] + pub from_snapshot_id: Option, + /// Target snapshot id. + pub to_snapshot_id: String, + /// Items added. + pub added: u32, + /// Items removed. + pub removed: u32, + /// Items modified. + pub modified: u32, + /// Items present and unchanged. + pub unchanged: u32, + /// Per-item changes. May be truncated by the driver; the counts above are + /// authoritative. + #[serde(default)] + pub changes: Vec, +} + +/// One item handed to [`crate::openhuman::memory::api::provider::MemorySourceSink`] by the host's sync +/// machinery. +/// +/// The host owns credentials, scheduling, and fetching; the driver owns storage +/// and indexing. This type is the whole of what crosses that line. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceItem { + /// Stable per-source item id. Dedupe key; not a display value. + pub item_id: String, + /// Display title. + #[serde(default)] + pub title: String, + /// Item body, already decoded to text. + pub content: String, + /// MIME type of [`Self::content`] when known. + #[serde(default)] + pub mime: Option, + /// Canonical URL back to the item, when it has one. + #[serde(default)] + pub url: Option, + /// Upstream last-modified time in milliseconds since the Unix epoch. + #[serde(default)] + pub updated_at_ms: Option, + /// Labels carried through from the source. + #[serde(default)] + pub tags: Vec, +} + +/// Outcome of one maintenance operation. +/// +/// A single shape covers reembed, compact, consolidate, and doctor because the +/// caller does the same thing with all four: report progress and surface +/// findings. A per-operation result type would multiply the contract surface +/// without giving any caller more to act on. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaintenanceReport { + /// Which operation ran (`reembed`, `compact`, `consolidate`, `doctor`). + pub operation: String, + /// Units the driver examined. + pub examined: u64, + /// Units the driver changed. Always `0` for `doctor`, which is read-only. + pub changed: u64, + /// Operator-facing findings and notes. Must not contain memory content or + /// credentials — this is logged and shown in status output. + #[serde(default)] + pub findings: Vec, +} + +#[cfg(test)] +#[path = "types_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/provider/types_tests.rs b/src/openhuman/memory/api/provider/types_tests.rs new file mode 100644 index 0000000000..390e59a24a --- /dev/null +++ b/src/openhuman/memory/api/provider/types_tests.rs @@ -0,0 +1,133 @@ +//! Tests for the contract-only value types. +//! +//! The focus is the two things a later slice can silently break: the +//! fail-closed reading of an empty [`SourceScope`], and the wire strings / +//! serde defaults that an out-of-process driver depends on. + +use super::*; + +#[test] +fn empty_source_scope_denies_every_source() { + let scope = SourceScope::default(); + assert!(scope.is_empty()); + assert!(!scope.allows_source_id("src-abc")); + assert!(!scope.allows_source_id("mem_src:src-abc:item")); +} + +#[test] +fn source_scope_matches_exact_id_and_mem_src_prefix() { + let scope = SourceScope::new(["src-abc", "src-def"]); + + assert!(scope.allows_source_id("src-abc")); + assert!(scope.allows_source_id("src-def")); + assert!(scope.allows_source_id("mem_src:src-abc:item-1")); + assert!(scope.allows_source_id("mem_src:src-def:nested:item")); + + assert!(!scope.allows_source_id("src-xyz")); + assert!(!scope.allows_source_id("mem_src:src-xyz:item-1")); +} + +#[test] +fn source_scope_prefix_requires_the_trailing_separator() { + // `src-abc` must not smear onto `src-abcdef`: the engine's SQL binds + // `mem_src:{id}:` including the trailing colon, so a longer id that merely + // starts with an allowed one is out of scope. + let scope = SourceScope::new(["src-abc"]); + assert!(!scope.allows_source_id("mem_src:src-abcdef:item")); + assert!(!scope.allows_source_id("src-abcdef")); +} + +#[test] +fn change_kind_wire_strings_match_the_engine() { + // These strings are shared with `memory::diff::types::ChangeKind`, so the + // embedded adapter maps rather than translates. Changing one is a contract + // major bump. + for (kind, expected) in [ + (ChangeKind::Added, "added"), + (ChangeKind::Removed, "removed"), + (ChangeKind::Modified, "modified"), + ] { + assert_eq!(kind.as_str(), expected); + assert_eq!( + serde_json::to_value(kind).expect("serialize change kind"), + serde_json::Value::String(expected.to_string()), + ); + } +} + +#[test] +fn export_page_terminates_on_absent_cursor_not_empty_records() { + let page = ExportPage::default(); + assert!(page.records.is_empty()); + assert!(page.next_cursor.is_none()); + + // An empty page with a cursor is a legitimate mid-export state, so callers + // must not treat "no records" as the terminator. + let midway = ExportPage { + records: Vec::new(), + next_cursor: Some("cursor-2".to_string()), + }; + assert!(midway.next_cursor.is_some()); +} + +#[test] +fn export_record_round_trips_taint_and_opaque_payload() { + let record = ExportRecord { + kind: "vendor_specific_kind".to_string(), + id: "rec-1".to_string(), + namespace: Some("global".to_string()), + taint: MemoryTaint::ExternalSync, + payload: serde_json::json!({ "anything": [1, 2, 3] }), + }; + + let json = serde_json::to_string(&record).expect("serialize record"); + let back: ExportRecord = serde_json::from_str(&json).expect("deserialize record"); + + assert_eq!(back, record); + assert_eq!(back.taint, MemoryTaint::ExternalSync); +} + +#[test] +fn ingest_item_deserializes_from_the_minimal_body() { + // Every optional field carries `#[serde(default)]`, so a caller that knows + // only source, id, and content can still build a valid request. + let item: IngestItem = serde_json::from_value(serde_json::json!({ + "source": "notion", + "source_id": "page-1", + "content": "hello", + })) + .expect("deserialize minimal ingest item"); + + assert_eq!(item.source, DataSource::Notion); + assert_eq!(item.namespace, None); + assert_eq!(item.owner, ""); + assert!(item.tags.is_empty()); + // Provenance defaults to the conservative-for-writes `Internal`; the host + // guard overrides it explicitly on every sync path. + assert_eq!(item.taint, MemoryTaint::Internal); +} + +#[test] +fn maintenance_report_defaults_to_a_clean_read_only_run() { + let report = MaintenanceReport { + operation: "doctor".to_string(), + ..MaintenanceReport::default() + }; + assert_eq!(report.changed, 0); + assert!(report.findings.is_empty()); +} + +#[test] +fn diff_report_expresses_a_first_ever_diff_without_a_sentinel() { + let report = DiffReport { + source_id: "src-abc".to_string(), + from_snapshot_id: None, + to_snapshot_id: "snap-1".to_string(), + added: 3, + ..DiffReport::default() + }; + + let json = serde_json::to_value(&report).expect("serialize diff report"); + assert_eq!(json["from_snapshot_id"], serde_json::Value::Null); + assert_eq!(json["added"], 3); +} diff --git a/src/openhuman/memory/api/recall.rs b/src/openhuman/memory/api/recall.rs new file mode 100644 index 0000000000..20ef1ad6c2 --- /dev/null +++ b/src/openhuman/memory/api/recall.rs @@ -0,0 +1,157 @@ +//! Recall filter contracts — the borrowed engine form and the owned +//! contract/wire form, kept side by side so they cannot drift. +//! +//! ## Why there are two +//! +//! [`RecallOpts`] is the historical, engine-facing shape: it borrows its string +//! filters so a hot retrieval path allocates nothing. That makes it unusable as +//! a contract type in two independent ways — it derives no serde impls, so it +//! cannot be a `POST /v1/memory/recall` request body, and its lifetime +//! parameter would have to be threaded through every `#[async_trait]` recall +//! method, which destroys the object safety the whole driver model rests on. +//! +//! [`OwnedRecallOpts`] is the answer: the same five fields, owned, serde- +//! derived. Contract and wire paths use the owned form; the engine path keeps +//! the borrowed one and converts at the boundary via +//! `RecallOpts::from(&owned)`, which is zero-copy for the string fields. +//! +//! ## Field parity is the contract +//! +//! A field added to one form and not the other is a silent contract hole: the +//! wire would accept a filter the engine never applies, or the engine would +//! offer a filter no remote driver can be told about. Two defences are in +//! place, and both must stay: +//! +//! 1. Both [`From`] impls **exhaustively destructure** their source, so adding +//! a field to either struct without handling it fails to compile. +//! 2. `owned_and_borrowed_recall_opts_have_identical_fields` in +//! `recall_tests.rs` round-trips a fully non-default value through both +//! directions, so a field that is merely *dropped* during conversion fails +//! the test. +//! +//! Both types live in this module (rather than in `types.rs`) precisely so the +//! pair is read and edited together. They are re-exported from +//! [`crate::openhuman::memory::api::types`], so every historical `types::RecallOpts` path — including +//! the engine crate's `crate::openhuman::memory::engine::types::` alias — keeps resolving. + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::memory::api::types::MemoryCategory; + +/// Optional filters for recall — the **borrowed, engine-facing** form. +/// +/// Borrows its string filters so an engine call path can pass slices of a +/// caller-owned request without allocating. It is deliberately *not* +/// serializable and deliberately *not* used in the driver contract: a lifetime +/// parameter cannot travel through an object-safe `#[async_trait]` method, and +/// a borrowed struct cannot be a request body. +/// +/// Use [`OwnedRecallOpts`] for anything that crosses a trait object or the +/// wire, and convert at the boundary with the [`From`] impl below. The two +/// types carry the same fields; a field added to one and not the other is a +/// silent contract hole, which +/// `owned_and_borrowed_recall_opts_have_identical_fields` exists to catch. +#[derive(Debug, Default, Clone)] +pub struct RecallOpts<'a> { + /// Restrict recall to this namespace; `None` falls back to [`crate::openhuman::memory::api::types::GLOBAL_NAMESPACE`]. + pub namespace: Option<&'a str>, + /// Restrict recall to entries of this category. + pub category: Option, + /// Restrict recall to entries scoped to this session. + pub session_id: Option<&'a str>, + /// Drop hits scoring below this threshold (typically 0.0–1.0). + pub min_score: Option, + /// When `true`, include conversational hits from other sessions in the same + /// workspace alongside the namespace recall. + pub cross_session: bool, +} + +/// Optional filters for recall — the **owned, contract-facing** form. +/// +/// This is the type the driver contract and the JSON wire protocol use. It +/// exists because [`RecallOpts`] cannot serve either role: +/// +/// - it derives no `Serialize`/`Deserialize`, so it cannot be a +/// `POST /v1/memory/recall` request body; +/// - it carries a borrow lifetime, which would have to be threaded through +/// every `#[async_trait]` recall method and destroys object safety at the +/// `dyn` boundary the whole driver model rests on. +/// +/// The borrowed form stays for engine-internal use so the embedded driver's +/// hot path allocates nothing: build the owned value once at the contract +/// boundary, then hand `RecallOpts::from(&owned)` down. +/// +/// Every field is `#[serde(default)]` so a minimal request body — even `{}` — +/// deserializes to the same value as [`Default::default`]. +#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] +pub struct OwnedRecallOpts { + /// Restrict recall to this namespace; `None` falls back to [`crate::openhuman::memory::api::types::GLOBAL_NAMESPACE`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace: Option, + /// Restrict recall to entries of this category. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub category: Option, + /// Restrict recall to entries scoped to this session. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + /// Drop hits scoring below this threshold (typically 0.0–1.0). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_score: Option, + /// When `true`, include conversational hits from other sessions in the same + /// workspace alongside the namespace recall. + #[serde(default)] + pub cross_session: bool, +} + +impl<'a> From<&'a OwnedRecallOpts> for RecallOpts<'a> { + /// Borrows the owned form for an engine call. Zero-copy for the two string + /// fields; [`MemoryCategory`] is cloned because it owns a `String` in its + /// [`MemoryCategory::Custom`] variant and [`RecallOpts`] holds it by value. + /// + /// Exhaustively destructures the source so adding a field to + /// [`OwnedRecallOpts`] without handling it here is a compile error. + fn from(owned: &'a OwnedRecallOpts) -> Self { + let OwnedRecallOpts { + namespace, + category, + session_id, + min_score, + cross_session, + } = owned; + RecallOpts { + namespace: namespace.as_deref(), + category: category.clone(), + session_id: session_id.as_deref(), + min_score: *min_score, + cross_session: *cross_session, + } + } +} + +impl From> for OwnedRecallOpts { + /// Takes ownership of a borrowed form — the direction a transport adapter + /// needs when turning an engine-shaped call into a request body. + /// + /// Exhaustively destructures the source for the same reason as the inverse + /// impl. + fn from(borrowed: RecallOpts<'_>) -> Self { + let RecallOpts { + namespace, + category, + session_id, + min_score, + cross_session, + } = borrowed; + OwnedRecallOpts { + namespace: namespace.map(str::to_string), + category, + session_id: session_id.map(str::to_string), + min_score, + cross_session, + } + } +} + +#[cfg(test)] +#[path = "recall_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/recall_tests.rs b/src/openhuman/memory/api/recall_tests.rs new file mode 100644 index 0000000000..5adf31754c --- /dev/null +++ b/src/openhuman/memory/api/recall_tests.rs @@ -0,0 +1,158 @@ +//! Unit tests for the recall filter contracts in [`super`]. +//! +//! The load-bearing test here is +//! `owned_and_borrowed_recall_opts_have_identical_fields`: it is the runtime +//! half of the field-parity defence described in the module docs (the compile +//! half being the exhaustive destructuring inside both `From` impls). + +use super::*; +use serde_json::json; + +/// Every field set to a non-default value, so a conversion that silently drops +/// one is visible. +fn fully_populated_owned() -> OwnedRecallOpts { + OwnedRecallOpts { + namespace: Some("projects".to_string()), + category: Some(MemoryCategory::Custom("field_notes".to_string())), + session_id: Some("session-42".to_string()), + min_score: Some(0.75), + cross_session: true, + } +} + +#[test] +fn owned_and_borrowed_recall_opts_have_identical_fields() { + let owned = fully_populated_owned(); + + // Owned → borrowed. Destructured exhaustively so a new field on + // `RecallOpts` fails to compile here rather than silently going unchecked. + let borrowed = RecallOpts::from(&owned); + let RecallOpts { + namespace, + category, + session_id, + min_score, + cross_session, + } = borrowed.clone(); + assert_eq!(namespace, Some("projects")); + assert_eq!(category, Some(MemoryCategory::Custom("field_notes".into()))); + assert_eq!(session_id, Some("session-42")); + assert_eq!(min_score, Some(0.75)); + assert!(cross_session); + + // Borrowed → owned, and back to the value we started from. A field dropped + // in either direction fails this equality. + let round_tripped = OwnedRecallOpts::from(borrowed); + assert_eq!(round_tripped, owned); +} + +#[test] +fn borrowed_view_is_zero_copy_over_the_owned_strings() { + let owned = fully_populated_owned(); + let borrowed = RecallOpts::from(&owned); + + // The borrowed form points *into* the owned value rather than at a copy; + // that is the whole reason the borrowed form survives. + assert_eq!( + borrowed.namespace.unwrap().as_ptr(), + owned.namespace.as_deref().unwrap().as_ptr() + ); + assert_eq!( + borrowed.session_id.unwrap().as_ptr(), + owned.session_id.as_deref().unwrap().as_ptr() + ); +} + +#[test] +fn owned_recall_opts_defaults_match_borrowed_defaults() { + let owned = OwnedRecallOpts::default(); + let borrowed = RecallOpts::from(&owned); + + assert!(borrowed.namespace.is_none()); + assert!(borrowed.category.is_none()); + assert!(borrowed.session_id.is_none()); + assert!(borrowed.min_score.is_none()); + assert!(!borrowed.cross_session); + + // And the borrowed default converts back to the owned default. + assert_eq!(OwnedRecallOpts::from(RecallOpts::default()), owned); +} + +#[test] +fn owned_recall_opts_serde_round_trips_every_field() { + let owned = fully_populated_owned(); + let encoded = serde_json::to_value(&owned).unwrap(); + + assert_eq!( + encoded, + json!({ + "namespace": "projects", + "category": "custom:field_notes", + "session_id": "session-42", + "min_score": 0.75, + "cross_session": true + }) + ); + + let decoded: OwnedRecallOpts = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded, owned); +} + +#[test] +fn empty_recall_body_deserializes_to_the_default() { + // A minimal `POST /v1/memory/recall` body must be accepted: every field is + // `#[serde(default)]`. + let decoded: OwnedRecallOpts = serde_json::from_value(json!({})).unwrap(); + assert_eq!(decoded, OwnedRecallOpts::default()); +} + +#[test] +fn partial_recall_body_leaves_unmentioned_fields_at_default() { + let decoded: OwnedRecallOpts = + serde_json::from_value(json!({ "namespace": "global" })).unwrap(); + assert_eq!(decoded.namespace.as_deref(), Some("global")); + assert!(decoded.category.is_none()); + assert!(decoded.session_id.is_none()); + assert!(decoded.min_score.is_none()); + assert!(!decoded.cross_session); +} + +/// The wire form omits absent filters rather than emitting explicit nulls. +/// +/// `OwnedRecallOpts` is the body of `POST /v1/memory/recall`, which the spec +/// describes as an optional-filters bag. Emitting `"namespace": null` for every +/// unset filter is valid JSON but forces a backend to distinguish "absent" from +/// "explicitly null" for no gain. Pinned here because changing the emitted shape +/// after a driver has shipped is observable to any backend that draws that +/// distinction. +#[test] +fn absent_recall_filters_are_omitted_from_the_wire_form() { + let json = serde_json::to_value(OwnedRecallOpts::default()).expect("serialize"); + assert_eq!( + json, + serde_json::json!({ "cross_session": false }), + "unset optional filters must be omitted, not serialized as null" + ); + + let populated = OwnedRecallOpts { + namespace: Some("work".into()), + ..Default::default() + }; + let json = serde_json::to_value(&populated).expect("serialize"); + assert_eq!( + json, + serde_json::json!({ "namespace": "work", "cross_session": false }) + ); +} + +/// Omitting a filter and sending it as `null` must both decode to `None`, so a +/// backend built against either spelling keeps working. +#[test] +fn omitted_and_explicit_null_recall_filters_both_decode_to_none() { + let omitted: OwnedRecallOpts = serde_json::from_str("{}").expect("decode {}"); + let explicit: OwnedRecallOpts = + serde_json::from_str(r#"{"namespace":null,"category":null,"session_id":null,"min_score":null,"cross_session":false}"#) + .expect("decode explicit nulls"); + assert_eq!(omitted, explicit); + assert_eq!(omitted, OwnedRecallOpts::default()); +} diff --git a/src/openhuman/memory/api/tool_memory.rs b/src/openhuman/memory/api/tool_memory.rs new file mode 100644 index 0000000000..8699d6d9f1 --- /dev/null +++ b/src/openhuman/memory/api/tool_memory.rs @@ -0,0 +1,162 @@ +//! Domain types for the tool-scoped memory layer. +//! +//! A [`ToolMemoryRule`] is a durable, actionable instruction attached to a +//! specific tool (e.g. `email`, `shell`, `web_search`). Unlike per-tool +//! effectiveness statistics, these rules capture **guidance** — corrections, +//! safety constraints, and learned operational rules that the agent should +//! obey when considering or invoking that tool. +//! +//! Rules carry a [`ToolMemoryPriority`] level so the retrieval pipeline can +//! distinguish safety-critical instructions from soft suggestions: +//! +//! - [`ToolMemoryPriority::Critical`] — pinned into the system prompt and +//! therefore not subject to mid-session context compression. +//! - [`ToolMemoryPriority::High`] — surfaced alongside critical rules at +//! tool-selection time. +//! - [`ToolMemoryPriority::Normal`] — available on demand via the recall +//! APIs, but not eagerly injected. +//! +//! These are pure data contracts: the snake_case wire strings +//! (`normal`/`high`/`critical`, `user_explicit`/`post_turn`/`programmatic`) +//! are preserved verbatim from OpenHuman so serialized rules stay +//! byte-compatible across the boundary. + +use serde::{Deserialize, Serialize}; + +/// Priority/criticality of a [`ToolMemoryRule`]. +/// +/// Used by both storage (to filter what is pinned into the system prompt) +/// and retrieval (to sort high-priority guidance ahead of advisory notes). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[derive(Default)] +pub enum ToolMemoryPriority { + /// Soft suggestion — surfaced on demand, not eagerly injected. + #[default] + Normal, + /// Important guidance — eagerly injected at tool-selection time. + High, + /// Safety-critical rule — pinned into the (compression-resistant) + /// system prompt so it survives the agent's full session. + Critical, +} + +impl ToolMemoryPriority { + /// True for priorities that must be eagerly surfaced to the agent + /// (Critical/High rules are both pinned into the system prompt and + /// prefetched at session start, so they survive context compression). + pub fn is_eager(self) -> bool { + matches!(self, Self::Critical | Self::High) + } +} + +/// Where a [`ToolMemoryRule`] originated from. +/// +/// Recorded for provenance and so consumers (UI / debugging) can tell user +/// edicts apart from auto-captured observations. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[derive(Default)] +pub enum ToolMemorySource { + /// User explicitly asked the agent to remember this rule. + UserExplicit, + /// Captured automatically from a post-turn observation (tool failure, + /// repeated correction, etc.). + PostTurn, + /// Written by another subsystem (e.g. an integration provisioner). + #[default] + Programmatic, +} + +/// A single tool-scoped memory rule. +/// +/// Stored under the `tool-{tool_name}` namespace as an entry keyed by +/// `rule/{rule_id}`. The id is stable across updates so callers can +/// upsert by replaying the same id. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolMemoryRule { + /// Stable identifier within `(tool_name)`. Generated by callers via + /// [`ToolMemoryRule::generate_id`] when one is not supplied. + pub id: String, + /// Tool this rule applies to (e.g. `email`, `shell`). + pub tool_name: String, + /// Natural-language guidance that should reach the agent. + pub rule: String, + /// Criticality level for retrieval and compression behaviour. + #[serde(default)] + pub priority: ToolMemoryPriority, + /// Where this rule came from. + #[serde(default)] + pub source: ToolMemorySource, + /// Optional free-form tags for filtering (e.g. `safety`, `permission`). + #[serde(default)] + pub tags: Vec, + /// RFC3339 timestamp of when the rule was first written. + pub created_at: String, + /// RFC3339 timestamp of the last update. + pub updated_at: String, +} + +impl ToolMemoryRule { + /// Build a new rule with a freshly generated id and `created_at` / + /// `updated_at` set to "now". + pub fn new( + tool_name: impl Into, + rule: impl Into, + priority: ToolMemoryPriority, + source: ToolMemorySource, + ) -> Self { + let now = chrono::Utc::now().to_rfc3339(); + Self { + id: Self::generate_id(), + tool_name: tool_name.into(), + rule: rule.into(), + priority, + source, + tags: Vec::new(), + created_at: now.clone(), + updated_at: now, + } + } + + /// Generate a fresh, opaque rule id. + /// + /// Each byte of a v4 UUID is encoded as two lowercase ASCII letters in + /// the `a..=p` range (one per nibble). The result is a separator-free, + /// digit-free token — deliberately shaped so it never trips a PII + /// boundary check when used as a storage key. + pub fn generate_id() -> String { + let mut id = String::with_capacity(33); + id.push('r'); + for byte in uuid::Uuid::new_v4().as_bytes() { + id.push((b'a' + (byte >> 4)) as char); + id.push((b'a' + (byte & 0x0f)) as char); + } + id + } + + /// Storage key used inside the tool namespace. + pub fn storage_key(id: &str) -> String { + format!("rule/{id}") + } +} + +/// Namespace string for a given tool. Trimmed and lower-cased so callers +/// can pass user-supplied tool names without leaking whitespace into +/// downstream queries. +/// +/// The `tool-` prefix is intentionally distinct from `global`, `skill-…` +/// and `tool_effectiveness` so retrieval and clearing operations can +/// reason about the namespace without ambiguity. Always build the +/// namespace through this helper — never hard-code the `tool-` format. +/// +/// The engine crate's `ToolMemoryStore::put_rule` applies the same +/// normalization to the stored rule so namespace and display/grouping identity +/// cannot diverge. +pub fn tool_memory_namespace(tool_name: &str) -> String { + format!("tool-{}", tool_name.trim().to_lowercase()) +} + +#[cfg(test)] +#[path = "tool_memory_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/tool_memory_tests.rs b/src/openhuman/memory/api/tool_memory_tests.rs new file mode 100644 index 0000000000..821369eaed --- /dev/null +++ b/src/openhuman/memory/api/tool_memory_tests.rs @@ -0,0 +1,128 @@ +//! Tests for the tool-scoped memory domain types. + +use super::*; + +#[test] +fn priority_default_is_normal() { + assert_eq!(ToolMemoryPriority::default(), ToolMemoryPriority::Normal); +} + +#[test] +fn priority_ordering_puts_critical_above_high() { + assert!(ToolMemoryPriority::Critical > ToolMemoryPriority::High); + assert!(ToolMemoryPriority::High > ToolMemoryPriority::Normal); +} + +#[test] +fn priority_is_eager_for_high_and_critical_only() { + assert!(ToolMemoryPriority::Critical.is_eager()); + assert!(ToolMemoryPriority::High.is_eager()); + assert!(!ToolMemoryPriority::Normal.is_eager()); +} + +#[test] +fn priority_snake_case_serde() { + assert_eq!( + serde_json::to_string(&ToolMemoryPriority::Critical).unwrap(), + "\"critical\"" + ); + assert_eq!( + serde_json::to_string(&ToolMemoryPriority::Normal).unwrap(), + "\"normal\"" + ); +} + +#[test] +fn source_snake_case_serde() { + assert_eq!( + serde_json::to_string(&ToolMemorySource::UserExplicit).unwrap(), + "\"user_explicit\"" + ); + assert_eq!( + serde_json::to_string(&ToolMemorySource::PostTurn).unwrap(), + "\"post_turn\"" + ); + assert_eq!( + serde_json::to_string(&ToolMemorySource::Programmatic).unwrap(), + "\"programmatic\"" + ); +} + +#[test] +fn source_default_is_programmatic() { + assert_eq!(ToolMemorySource::default(), ToolMemorySource::Programmatic); +} + +#[test] +fn rule_new_fills_id_and_timestamps() { + let rule = ToolMemoryRule::new( + "email", + "never email Sarah", + ToolMemoryPriority::Critical, + ToolMemorySource::UserExplicit, + ); + assert!(!rule.id.is_empty()); + assert_eq!(rule.tool_name, "email"); + assert_eq!(rule.rule, "never email Sarah"); + assert_eq!(rule.priority, ToolMemoryPriority::Critical); + assert_eq!(rule.source, ToolMemorySource::UserExplicit); + assert!(rule.created_at == rule.updated_at); +} + +#[test] +fn rule_generate_id_produces_unique_values() { + let a = ToolMemoryRule::generate_id(); + let b = ToolMemoryRule::generate_id(); + assert_ne!(a, b); + assert!(a.starts_with('r')); + assert!(a[1..].chars().all(|c| matches!(c, 'a'..='p'))); +} + +#[test] +fn generated_rule_ids_are_safe_memory_document_keys() { + // Generated ids must be free of digits and separators so the resulting + // storage key never resembles PII (phone numbers, ids, etc.) to a + // boundary check downstream. + for _ in 0..128 { + let id = ToolMemoryRule::generate_id(); + assert!( + id.chars().all(|ch| ch.is_ascii_lowercase()), + "generated id should avoid PII-shaped digits and separators: {id}" + ); + let key = ToolMemoryRule::storage_key(&id); + assert!( + key.bytes().all(|b| b == b'/' || b.is_ascii_lowercase()), + "generated storage key should not contain PII-shaped bytes: {key}" + ); + } +} + +#[test] +fn rule_storage_key_uses_rule_prefix() { + assert_eq!(ToolMemoryRule::storage_key("abc"), "rule/abc"); +} + +#[test] +fn rule_serde_roundtrip_preserves_fields() { + let rule = ToolMemoryRule { + id: "id-1".into(), + tool_name: "shell".into(), + rule: "never run sudo".into(), + priority: ToolMemoryPriority::High, + source: ToolMemorySource::PostTurn, + tags: vec!["safety".into()], + created_at: "2026-05-11T00:00:00Z".into(), + updated_at: "2026-05-11T00:00:01Z".into(), + }; + let json = serde_json::to_string(&rule).unwrap(); + let back: ToolMemoryRule = serde_json::from_str(&json).unwrap(); + assert_eq!(back, rule); +} + +#[test] +fn namespace_uses_tool_prefix_and_trims_whitespace() { + assert_eq!(tool_memory_namespace("email"), "tool-email"); + assert_eq!(tool_memory_namespace(" shell "), "tool-shell"); + assert_eq!(tool_memory_namespace("Send_Email"), "tool-send_email"); + assert_eq!(tool_memory_namespace("WebSearch"), "tool-websearch"); +} diff --git a/src/openhuman/memory/api/traits.rs b/src/openhuman/memory/api/traits.rs new file mode 100644 index 0000000000..c96e0b9bd8 --- /dev/null +++ b/src/openhuman/memory/api/traits.rs @@ -0,0 +1,157 @@ +//! The high-level [`Memory`] trait every storage backend implements. +//! +//! Ported from OpenHuman's `memory::traits`. Backend-specific escape hatches +//! (e.g. raw SQLite connection access) are intentionally omitted here so the +//! trait stays storage-agnostic; concrete backends expose those via their own +//! inherent methods. +//! +//! ## Contract notes +//! +//! - Every method returns `anyhow::Result<_>` rather than a typed error: this +//! trait is a stable abstraction boundary over heterogeneous backends +//! (SQLite, vector DB, in-memory, …), each with its own error domain, so +//! callers should treat a returned `Err` as opaque and log/propagate it +//! rather than match on its variant. Concrete backends document their own +//! failure modes (e.g. IO errors, malformed persisted rows) alongside their +//! inherent methods. +//! - None of these methods are specified to panic; a conforming implementation +//! should convert failures (invalid input, backend errors, poisoned locks) +//! into `Err` instead. +//! - [`Memory::store`] and [`Memory::store_with_taint`] are upserts keyed by +//! `(namespace, key)`: calling them again with the same key replaces the +//! prior entry rather than erroring or duplicating it. + +use async_trait::async_trait; + +use super::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts}; + +/// The core trait for memory storage and retrieval. +/// +/// Any persistence backend (SQLite, Postgres, vector DB, in-memory, …) should +/// implement this to participate in a TinyMemory-backed memory engine. +#[async_trait] +pub trait Memory: Send + Sync { + /// Returns the backend name (e.g. `"sqlite"`, `"vector"`, `"in_memory"`). + fn name(&self) -> &str; + + /// Stores a new memory entry or updates an existing one. + /// + /// Idempotent upsert keyed by `(namespace, key)`: calling this again with + /// the same `namespace`/`key` replaces the previous `content`, `category`, + /// and `session_id` rather than erroring or creating a duplicate. Entries + /// stored this way carry [`MemoryTaint::Internal`] (the default); use + /// [`Self::store_with_taint`] to persist content from an external source. + /// + /// # Errors + /// + /// Returns `Err` on any backend failure (IO, serialization, connection + /// loss); implementations must not panic on caller-controlled input. + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + ) -> anyhow::Result<()>; + + /// Store an entry with explicit provenance taint. + /// + /// Sync paths ingesting third-party text MUST use this with + /// [`MemoryTaint::ExternalSync`]. The default implementation degrades to + /// [`Self::store`] for backends that do not yet persist taint — meaning it + /// silently drops the `taint` argument for any backend that has not + /// overridden this method. Backends whose durability/policy story depends + /// on taint being recorded MUST override this method rather than rely on + /// the default. + async fn store_with_taint( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> anyhow::Result<()> { + if taint != MemoryTaint::Internal { + anyhow::bail!("backend does not support taint-preserving storage"); + } + self.store(namespace, key, content, category, session_id) + .await + } + + /// Recalls memories matching a query using keyword or semantic search. + /// + /// `limit` caps the number of returned entries; `opts` narrows the search + /// by namespace, category, session, minimum score, and cross-session + /// inclusion (see [`RecallOpts`]). An empty or non-matching `query` should + /// yield `Ok(vec![])`, not an error. Result ordering is backend-defined + /// (typically most-relevant first) but callers must not assume a stable + /// order across backends. + async fn recall( + &self, + query: &str, + limit: usize, + opts: RecallOpts<'_>, + ) -> anyhow::Result>; + + /// Recall documents whose *vector* similarity alone meets a threshold. + /// + /// Returns `(key, content)` pairs, most-relevant first. Defaults to empty so + /// keyword-only / mock backends opt out; a backend that overrides this + /// should treat `min_vector_similarity` as an inclusive floor (hits scoring + /// strictly below it are dropped) and `limit` as a hard cap on the + /// returned count. + async fn recall_relevant_by_vector( + &self, + namespace: &str, + query: &str, + limit: usize, + min_vector_similarity: f64, + ) -> anyhow::Result> { + let _ = (namespace, query, limit, min_vector_similarity); + Ok(Vec::new()) + } + + /// Retrieves a specific entry by exact `(namespace, key)`. + /// + /// Returns `Ok(None)` — not `Err` — when no entry exists for the pair; + /// `Err` is reserved for backend failures. + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result>; + + /// Lists entries, optionally scoped by namespace, category, and session. + /// + /// Each `Option` filter narrows the result set when `Some`; passing all + /// three as `None` lists every entry the backend holds. An empty result + /// set is `Ok(vec![])`, never an error. + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> anyhow::Result>; + + /// Deletes the entry for `(namespace, key)`. Returns whether it existed. + /// + /// Idempotent: forgetting an already-absent `(namespace, key)` returns + /// `Ok(false)` rather than erroring, so callers may call this + /// unconditionally without checking existence first. + async fn forget(&self, namespace: &str, key: &str) -> anyhow::Result; + + /// Lists all namespaces with aggregate stats for agent-side discovery. + /// + /// See [`NamespaceSummary`] for the per-namespace count and + /// last-updated timestamp returned. + async fn namespace_summaries(&self) -> anyhow::Result>; + + /// Total count of all entries in the backend, across all namespaces. + async fn count(&self) -> anyhow::Result; + + /// Health check on the underlying storage system. + /// + /// Returns `true` when the backend is reachable and able to serve + /// requests. Unlike the other methods this reports failure as `false` + /// rather than `Err`, so it is safe to call from a liveness probe without + /// error-handling boilerplate. + async fn health_check(&self) -> bool; +} diff --git a/src/openhuman/memory/api/tree.rs b/src/openhuman/memory/api/tree.rs new file mode 100644 index 0000000000..b6c6bd5576 --- /dev/null +++ b/src/openhuman/memory/api/tree.rs @@ -0,0 +1,212 @@ +//! Domain types for the markdown time-based summary tree. +//! +//! Organises summaries as a time hierarchy: root → year → month → day → hour +//! (leaf). Ported from OpenHuman's `memory_tree/tree_runtime/types.rs`. + +use chrono::{DateTime, Datelike, Timelike, Utc}; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +/// Hierarchical level of a tree node. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NodeLevel { + /// Single tree root; aggregates all years. Wire string `"root"`. + Root, + /// One node per calendar year. Wire string `"year"`. + Year, + /// One node per calendar month. Wire string `"month"`. + Month, + /// One node per calendar day. Wire string `"day"`. + Day, + /// Leaf level; one node per hour, where raw content lands. Wire string `"hour"`. + Hour, +} + +impl NodeLevel { + /// Maximum number of tokens allowed at this level. + pub fn max_tokens(&self) -> u32 { + match self { + Self::Hour => 1_000, + Self::Day => 2_000, + Self::Month => 4_000, + Self::Year => 8_000, + Self::Root => 20_000, + } + } + + /// The level above this one in the hierarchy (`None` for root). + pub fn parent_level(&self) -> Option { + match self { + Self::Hour => Some(Self::Day), + Self::Day => Some(Self::Month), + Self::Month => Some(Self::Year), + Self::Year => Some(Self::Root), + Self::Root => None, + } + } + + /// True only for the leaf level (hour). + pub fn is_leaf(&self) -> bool { + matches!(self, Self::Hour) + } + + /// Parse a level string from YAML frontmatter. + pub fn from_str_label(s: &str) -> Option { + match s.trim().to_ascii_lowercase().as_str() { + "root" => Some(Self::Root), + "year" => Some(Self::Year), + "month" => Some(Self::Month), + "day" => Some(Self::Day), + "hour" => Some(Self::Hour), + _ => None, + } + } + + /// Label for display / frontmatter. + pub fn as_str(&self) -> &'static str { + match self { + Self::Root => "root", + Self::Year => "year", + Self::Month => "month", + Self::Day => "day", + Self::Hour => "hour", + } + } +} + +/// A single node in the summary tree. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TreeNode { + /// Path-style hierarchical id, e.g. `"2024/03/15/09"` or `"root"`. + pub node_id: String, + /// Namespace owning this tree (isolates independent trees). + pub namespace: String, + /// Hierarchical level this node sits at. + pub level: NodeLevel, + /// Id of the parent node; `None` only for the root. + pub parent_id: Option, + /// Rolled-up summary text for this node. + pub summary: String, + /// Estimated token count of [`Self::summary`]; bounded by [`NodeLevel::max_tokens`]. + pub token_count: u32, + /// Number of direct children rolled into this node. + pub child_count: u32, + /// Creation timestamp (UTC). + pub created_at: DateTime, + /// Last-update timestamp (UTC). + pub updated_at: DateTime, + /// Optional opaque metadata blob; omitted from serialization when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, +} + +/// Metadata about an entire tree within a namespace. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TreeStatus { + /// Namespace the tree belongs to. + pub namespace: String, + /// Total number of nodes across all levels. + pub total_nodes: u64, + /// Number of populated levels (tree height). + pub depth: u32, + /// Timestamp of the earliest ingested entry, if any. + pub oldest_entry: Option>, + /// Timestamp of the most recent ingested entry, if any. + pub newest_entry: Option>, + /// When the tree was last (re)built or sealed. + pub last_run_at: Option>, +} + +/// Input for appending raw content to the ingestion buffer. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IngestRequest { + /// Target namespace to append content into. + pub namespace: String, + /// Raw content to buffer for summarization. + pub content: String, + /// Event time used to derive the hour leaf; defaults to ingestion time when absent. + #[serde(default)] + pub timestamp: Option>, + /// Optional structured metadata carried alongside the content. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, +} + +/// Result of a tree query at a specific node. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryResult { + /// The node addressed by the query. + pub node: TreeNode, + /// Direct children of [`Self::node`], for drill-down navigation. + pub children: Vec, +} + +/// Rough token estimate: ~4 characters per token. +pub fn estimate_tokens(text: &str) -> u32 { + u32::try_from(text.len().div_ceil(4)).unwrap_or(u32::MAX) +} + +/// Derive the parent node ID from a node ID. +pub fn derive_parent_id(node_id: &str) -> Option { + if node_id == "root" { + return None; + } + match node_id.rfind('/') { + Some(pos) => Some(node_id[..pos].to_string()), + None => Some("root".to_string()), + } +} + +/// Determine the `NodeLevel` from a node ID string. +pub fn level_from_node_id(node_id: &str) -> NodeLevel { + if node_id == "root" { + return NodeLevel::Root; + } + match node_id.matches('/').count() { + 0 => NodeLevel::Year, + 1 => NodeLevel::Month, + 2 => NodeLevel::Day, + _ => NodeLevel::Hour, + } +} + +/// Derive all ancestor node IDs from a timestamp (hour through root). +/// Returns `(hour_id, day_id, month_id, year_id, root_id)`. +pub fn derive_node_ids(ts: &DateTime) -> (String, String, String, String, String) { + let year = format!("{}", ts.year()); + let month = format!("{}/{:02}", ts.year(), ts.month()); + let day = format!("{}/{:02}/{:02}", ts.year(), ts.month(), ts.day()); + let hour = format!( + "{}/{:02}/{:02}/{:02}", + ts.year(), + ts.month(), + ts.day(), + ts.hour() + ); + (hour, day, month, year, "root".to_string()) +} + +/// Convert a node ID to a relative file path within the tree directory. +pub fn node_id_to_path(node_id: &str) -> PathBuf { + if node_id == "root" { + return PathBuf::from("root.md"); + } + if node_id.starts_with('/') + || node_id + .split('/') + .any(|part| part.is_empty() || !part.chars().all(|c| c.is_ascii_digit())) + { + return PathBuf::from("invalid"); + } + let level = level_from_node_id(node_id); + if level.is_leaf() { + PathBuf::from(format!("{node_id}.md")) + } else { + PathBuf::from(node_id).join("summary.md") + } +} + +#[cfg(test)] +#[path = "tree_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/tree_tests.rs b/src/openhuman/memory/api/tree_tests.rs new file mode 100644 index 0000000000..bb9965fa46 --- /dev/null +++ b/src/openhuman/memory/api/tree_tests.rs @@ -0,0 +1,86 @@ +//! Tests for the markdown time-tree node types. + +use super::*; +use chrono::TimeZone; +use std::path::PathBuf; + +#[test] +fn node_level_max_tokens() { + assert_eq!(NodeLevel::Hour.max_tokens(), 1_000); + assert_eq!(NodeLevel::Day.max_tokens(), 2_000); + assert_eq!(NodeLevel::Month.max_tokens(), 4_000); + assert_eq!(NodeLevel::Year.max_tokens(), 8_000); + assert_eq!(NodeLevel::Root.max_tokens(), 20_000); +} + +#[test] +fn node_level_parent_chain() { + assert_eq!(NodeLevel::Hour.parent_level(), Some(NodeLevel::Day)); + assert_eq!(NodeLevel::Day.parent_level(), Some(NodeLevel::Month)); + assert_eq!(NodeLevel::Month.parent_level(), Some(NodeLevel::Year)); + assert_eq!(NodeLevel::Year.parent_level(), Some(NodeLevel::Root)); + assert_eq!(NodeLevel::Root.parent_level(), None); +} + +#[test] +fn derive_parent_id_chain() { + assert_eq!(derive_parent_id("2024/03/15/14"), Some("2024/03/15".into())); + assert_eq!(derive_parent_id("2024/03/15"), Some("2024/03".into())); + assert_eq!(derive_parent_id("2024/03"), Some("2024".into())); + assert_eq!(derive_parent_id("2024"), Some("root".into())); + assert_eq!(derive_parent_id("root"), None); +} + +#[test] +fn level_from_node_id_all_levels() { + assert_eq!(level_from_node_id("root"), NodeLevel::Root); + assert_eq!(level_from_node_id("2024"), NodeLevel::Year); + assert_eq!(level_from_node_id("2024/03"), NodeLevel::Month); + assert_eq!(level_from_node_id("2024/03/15"), NodeLevel::Day); + assert_eq!(level_from_node_id("2024/03/15/14"), NodeLevel::Hour); +} + +#[test] +fn derive_node_ids_from_timestamp() { + let ts = Utc.with_ymd_and_hms(2024, 3, 15, 14, 30, 0).unwrap(); + let (hour, day, month, year, root) = derive_node_ids(&ts); + assert_eq!(hour, "2024/03/15/14"); + assert_eq!(day, "2024/03/15"); + assert_eq!(month, "2024/03"); + assert_eq!(year, "2024"); + assert_eq!(root, "root"); +} + +#[test] +fn node_id_to_path_mapping() { + assert_eq!(node_id_to_path("root"), PathBuf::from("root.md")); + assert_eq!(node_id_to_path("2024"), PathBuf::from("2024/summary.md")); + assert_eq!( + node_id_to_path("2024/03"), + PathBuf::from("2024/03/summary.md") + ); + assert_eq!( + node_id_to_path("2024/03/15/14"), + PathBuf::from("2024/03/15/14.md") + ); +} + +#[test] +fn estimate_tokens_rough() { + assert_eq!(estimate_tokens(""), 0); + assert_eq!(estimate_tokens("abcd"), 1); + assert_eq!(estimate_tokens(&"a".repeat(4000)), 1000); +} + +#[test] +fn node_level_roundtrip() { + for level in [ + NodeLevel::Root, + NodeLevel::Year, + NodeLevel::Month, + NodeLevel::Day, + NodeLevel::Hour, + ] { + assert_eq!(NodeLevel::from_str_label(level.as_str()), Some(level)); + } +} diff --git a/src/openhuman/memory/api/types.rs b/src/openhuman/memory/api/types.rs new file mode 100644 index 0000000000..913d23226c --- /dev/null +++ b/src/openhuman/memory/api/types.rs @@ -0,0 +1,436 @@ +//! Core public data contracts for the TinyMemory memory contract. +//! +//! These types are the stable surface shared across every layer (storage, +//! ingestion, retrieval, RPC). They are pure data — no storage side effects, +//! no interior mutability, freely `Clone`/`Send`/`Sync` — and are ported +//! faithfully from OpenHuman's `memory` and `memory_store` modules so wire +//! formats (snake_case enum strings, serde defaults) stay byte-compatible when +//! OpenHuman imports this crate. +//! +//! ## Wire-compatibility contract +//! +//! Every `#[serde(rename_all = "snake_case")]` enum here has its variant +//! strings persisted in on-disk indexes (SQLite columns, markdown frontmatter) +//! and/or sent over the RPC boundary. Renaming a variant, or a struct field +//! that lacks `#[serde(default)]`, is a breaking change for any host reading +//! previously-written data. When adding a field, prefer `#[serde(default)]` so +//! older persisted rows continue to deserialize. +//! +//! ## Fail-closed provenance +//! +//! [`MemoryTaint`] is the one field in this module with a safety-relevant +//! default: it decodes unknown/corrupt persisted strings as +//! [`MemoryTaint::ExternalSync`] rather than [`MemoryTaint::Internal`], so a +//! caller that forgets to persist taint, or an index that has drifted, fails +//! toward *more* restrictive tool-use policy rather than less. + +use serde::{Deserialize, Serialize}; + +/// The recall filter contracts live in [`crate::openhuman::memory::api::recall`] so the borrowed and +/// owned forms sit next to each other and cannot drift, and are re-exported +/// here so every historical `types::RecallOpts` path — including the engine +/// crate's `crate::openhuman::memory::engine::types::` alias — keeps resolving unchanged. +pub use crate::openhuman::memory::api::recall::{OwnedRecallOpts, RecallOpts}; + +/// Default namespace used when a caller passes no explicit namespace. +pub const GLOBAL_NAMESPACE: &str = "global"; + +/// Provenance / trust signal attached to a memory entry. +/// +/// Drives downstream policy — most importantly whether automation whose context +/// contains this content may invoke external-effect tools. Defaults to +/// [`MemoryTaint::Internal`] so legacy rows (no persisted taint column) and all +/// in-memory defaults are conservatively trusted as user-driven content. +/// +/// Sync paths that ingest text from third-party services (Gmail / Slack / +/// Notion / Composio / MCP / …) MUST set this to [`MemoryTaint::ExternalSync`] +/// at write time so callers can refuse external-effect tools on tainted context. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MemoryTaint { + /// User-driven memory (chat, manual remember, internal heuristics). + #[default] + Internal, + /// Content ingested from an external sync source. + ExternalSync, +} + +impl Serialize for MemoryTaint { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_db_str()) + } +} + +impl<'de> Deserialize<'de> for MemoryTaint { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = String::deserialize(deserializer)?; + Ok(Self::from_db_str(&raw)) + } +} + +impl MemoryTaint { + /// Serialised form used by the SQLite `memory_docs.taint` column. + /// + /// # Examples + /// + /// ``` + /// use crate::openhuman::memory::api::types::MemoryTaint; + /// + /// assert_eq!(MemoryTaint::Internal.as_db_str(), "internal"); + /// assert_eq!(MemoryTaint::ExternalSync.as_db_str(), "external_sync"); + /// ``` + pub fn as_db_str(&self) -> &'static str { + match self { + Self::Internal => "internal", + Self::ExternalSync => "external_sync", + } + } + + /// Reverse of [`Self::as_db_str`]. Unknown values fail closed to the more + /// restrictive [`MemoryTaint::ExternalSync`] so policy gates refuse + /// external-effect tools on content of unknown provenance. + /// + /// Note this is *not* a strict inverse of [`Self::as_db_str`]: it never + /// errors, so a malformed or unexpected `raw` string (empty, wrong case, + /// truncated by a partial write, …) silently maps to + /// [`MemoryTaint::ExternalSync`] rather than surfacing as a parse failure. + /// + /// # Examples + /// + /// ``` + /// use crate::openhuman::memory::api::types::MemoryTaint; + /// + /// assert_eq!(MemoryTaint::from_db_str("internal"), MemoryTaint::Internal); + /// assert_eq!(MemoryTaint::from_db_str("external_sync"), MemoryTaint::ExternalSync); + /// // Unrecognised input fails closed rather than erroring. + /// assert_eq!(MemoryTaint::from_db_str("garbage"), MemoryTaint::ExternalSync); + /// ``` + pub fn from_db_str(raw: &str) -> Self { + match raw { + "internal" => Self::Internal, + "external_sync" => Self::ExternalSync, + _ => Self::ExternalSync, + } + } +} + +/// Categories used to organize and filter memories by nature and lifecycle. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MemoryCategory { + /// Long-term foundational facts, user preferences, permanent decisions. + Core, + /// Temporal logs reflecting daily activities or ephemeral state. + Daily, + /// Contextual information derived from active conversations. + Conversation, + /// A user- or system-defined custom category. + Custom(String), +} + +/// The stable wire/display representation uses the built-in labels directly +/// and prefixes custom values with `custom:`. The prefix keeps +/// `Custom("core")` distinct from [`MemoryCategory::Core`] and makes Display, +/// serde, and [`std::str::FromStr`] true inverses. +impl std::fmt::Display for MemoryCategory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Core => write!(f, "core"), + Self::Daily => write!(f, "daily"), + Self::Conversation => write!(f, "conversation"), + Self::Custom(name) => write!(f, "custom:{name}"), + } + } +} + +impl std::str::FromStr for MemoryCategory { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "core" => Ok(Self::Core), + "daily" => Ok(Self::Daily), + "conversation" => Ok(Self::Conversation), + "custom:" => Ok(Self::Custom(String::new())), + value if value.starts_with("custom:") && value.len() > "custom:".len() => { + Ok(Self::Custom(value["custom:".len()..].to_string())) + } + value if !value.is_empty() => Ok(Self::Custom(value.to_string())), + _ => Err(format!("unknown memory category: {value}")), + } + } +} + +impl Serialize for MemoryCategory { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for MemoryCategory { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } +} + +/// A single stored memory entry with associated metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryEntry { + /// Unique identifier (usually a UUID). + pub id: String, + /// Key or title associated with this memory. + pub key: String, + /// Actual content / value of the memory. + pub content: String, + /// Optional namespace for logical separation. + #[serde(default)] + pub namespace: Option, + /// Organizational category. + pub category: MemoryCategory, + /// ISO 8601 timestamp of create / last-update. + pub timestamp: String, + /// Optional session scope. + pub session_id: Option, + /// Optional relevance / confidence score (typically 0.0–1.0). + pub score: Option, + /// Provenance taint (see [`MemoryTaint`]). Absent on legacy JSON, in which + /// case it defaults to [`MemoryTaint::Internal`]; unknown persisted string + /// values decode as [`MemoryTaint::ExternalSync`]. + #[serde(default)] + pub taint: MemoryTaint, +} + +/// Summary row for agent-side namespace discovery. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NamespaceSummary { + /// Namespace identifier. + pub namespace: String, + /// Number of memory entries currently stored in the namespace. + pub count: usize, + /// RFC3339 timestamp of the most recent update in the namespace, if any. + pub last_updated: Option, +} + +/// Input payload for upserting a namespace-scoped memory document. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NamespaceDocumentInput { + /// Target namespace for the document. + pub namespace: String, + /// Stable upsert key; reusing a key updates the existing document. + pub key: String, + /// Human-readable title. + pub title: String, + /// Document body. + pub content: String, + /// Origin of the content (e.g. `chat`, `gmail`, `notion`). + pub source_type: String, + /// Caller-defined priority label. + pub priority: String, + /// Free-form tags for filtering. + #[serde(default)] + pub tags: Vec, + /// Arbitrary structured metadata carried alongside the document. + #[serde(default)] + pub metadata: serde_json::Value, + /// Category label (see [`MemoryCategory`] wire strings). + pub category: String, + /// Optional session scope. + #[serde(default)] + pub session_id: Option, + /// Explicit document id; generated when absent. + #[serde(default)] + pub document_id: Option, + /// Provenance taint; defaults to [`MemoryTaint::Internal`] for legacy JSON + /// missing this field. Unknown persisted string values decode as + /// [`MemoryTaint::ExternalSync`]. + #[serde(default)] + pub taint: MemoryTaint, +} + +/// One ranked retrieval result for a namespace text query. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NamespaceQueryResult { + /// Upsert key of the matched document. + pub key: String, + /// Matched content. + pub content: String, + /// Relevance score for this hit. + pub score: f64, + /// Category label of the matched document. + pub category: String, + /// Provenance taint; unknown persisted values decode as `external_sync`. + #[serde(default)] + pub taint: MemoryTaint, +} + +/// Discriminator for the kind of stored memory item a hit refers to. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MemoryItemKind { + /// A namespace-scoped memory document (`memory_docs` row). + Document, + /// A key/value record. + Kv, + /// An episodic / conversational memory. + Episodic, + /// A discrete event entry. + Event, +} + +/// Persisted form of a memory document as stored in `memory_docs`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StoredMemoryDocument { + /// Unique document id. + pub document_id: String, + /// Owning namespace. + pub namespace: String, + /// Stable upsert key. + pub key: String, + /// Human-readable title. + pub title: String, + /// Document body. + pub content: String, + /// Origin of the content (e.g. `chat`, `gmail`). + pub source_type: String, + /// Caller-defined priority label. + pub priority: String, + /// Free-form tags. + pub tags: Vec, + /// Arbitrary structured metadata. + pub metadata: serde_json::Value, + /// Category label. + pub category: String, + /// Optional session scope. + pub session_id: Option, + /// Creation time as a Unix timestamp (seconds). + pub created_at: f64, + /// Last-update time as a Unix timestamp (seconds). + pub updated_at: f64, + /// Path, relative to the vault root, of the authoritative markdown file. + pub markdown_rel_path: String, + /// Provenance taint; unknown persisted values decode as `external_sync`. + #[serde(default)] + pub taint: MemoryTaint, +} + +/// A single KV row, namespace-scoped or global (when `namespace` is `None`). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryKvRecord { + /// Owning namespace, or `None` for a global row. + pub namespace: Option, + /// KV key. + pub key: String, + /// Stored JSON value. + pub value: serde_json::Value, + /// Last-update time as a Unix timestamp (seconds). + pub updated_at: f64, +} + +/// A graph edge (subject — predicate → object) plus accumulated evidence. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GraphRelationRecord { + /// Owning namespace, or `None` for a global relation. + pub namespace: Option, + /// Edge subject (head entity). + pub subject: String, + /// Relation type linking subject to object. + pub predicate: String, + /// Edge object (tail entity). + pub object: String, + /// Arbitrary structured attributes attached to the edge. + pub attrs: serde_json::Value, + /// Last-update time as a Unix timestamp (seconds). + pub updated_at: f64, + /// Number of independent observations supporting this edge. + pub evidence_count: u32, + /// Optional ordering hint among sibling relations. + pub order_index: Option, + /// Documents that contributed evidence for this edge. + pub document_ids: Vec, + /// Chunks that contributed evidence for this edge. + pub chunk_ids: Vec, +} + +/// Per-signal contribution to a hit's final score, for ranking explainers. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RetrievalScoreBreakdown { + /// Lexical / keyword match contribution. + pub keyword_relevance: f64, + /// Vector (cosine) similarity contribution. + pub vector_similarity: f64, + /// Graph-proximity contribution. + pub graph_relevance: f64, + /// Episodic-recall contribution. + pub episodic_relevance: f64, + /// Recency contribution. + pub freshness: f64, + /// Weighted combination of the above signals; the value used for ranking. + pub final_score: f64, +} + +/// A single ranked retrieval hit. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NamespaceMemoryHit { + /// Identifier of the matched item (interpretation depends on [`Self::kind`]). + pub id: String, + /// Which kind of stored item this hit refers to. + pub kind: MemoryItemKind, + /// Owning namespace. + pub namespace: String, + /// Upsert key of the matched item. + pub key: String, + /// Title, when the item has one. + pub title: Option, + /// Matched content. + pub content: String, + /// Category label. + pub category: String, + /// Origin of the content, when known. + pub source_type: Option, + /// Last-update time as a Unix timestamp (seconds). + pub updated_at: f64, + /// Final ranking score; mirrors [`RetrievalScoreBreakdown::final_score`]. + pub score: f64, + /// Per-signal explanation of how [`Self::score`] was derived. + pub score_breakdown: RetrievalScoreBreakdown, + /// Source document id, when the hit resolves to a document. + #[serde(default)] + pub document_id: Option, + /// Source chunk id, when the hit resolves to a chunk. + #[serde(default)] + pub chunk_id: Option, + /// Graph relations that reinforced this hit's ranking. + #[serde(default)] + pub supporting_relations: Vec, + /// Provenance taint; unknown persisted values decode as `external_sync`. + #[serde(default)] + pub taint: MemoryTaint, +} + +/// Aggregated retrieval result for a namespace: rendered context plus hits. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NamespaceRetrievalContext { + /// Namespace the retrieval ran against. + pub namespace: String, + /// Originating query text, if any. + pub query: Option, + /// Rendered, ready-to-inject context assembled from [`Self::hits`]. + pub context_text: String, + /// Ranked hits backing the rendered context. + pub hits: Vec, +} + +#[cfg(test)] +#[path = "types_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/types_tests.rs b/src/openhuman/memory/api/types_tests.rs new file mode 100644 index 0000000000..5ee61b5ac5 --- /dev/null +++ b/src/openhuman/memory/api/types_tests.rs @@ -0,0 +1,234 @@ +//! Unit tests for the core memory data contracts in [`super`]. + +use super::*; +use serde_json::json; + +#[test] +fn global_namespace_constant_is_stable() { + assert_eq!(GLOBAL_NAMESPACE, "global"); +} + +#[test] +fn memory_category_display_outputs_expected_values() { + assert_eq!(MemoryCategory::Core.to_string(), "core"); + assert_eq!(MemoryCategory::Daily.to_string(), "daily"); + assert_eq!(MemoryCategory::Conversation.to_string(), "conversation"); + assert_eq!( + MemoryCategory::Custom("project_notes".into()).to_string(), + "custom:project_notes" + ); +} + +#[test] +fn memory_category_serde_uses_snake_case() { + assert_eq!( + serde_json::to_string(&MemoryCategory::Core).unwrap(), + "\"core\"" + ); + assert_eq!( + serde_json::to_string(&MemoryCategory::Daily).unwrap(), + "\"daily\"" + ); + assert_eq!( + serde_json::to_string(&MemoryCategory::Conversation).unwrap(), + "\"conversation\"" + ); + assert_eq!( + serde_json::to_string(&MemoryCategory::Custom("core".into())).unwrap(), + "\"custom:core\"" + ); + for category in [ + MemoryCategory::Core, + MemoryCategory::Daily, + MemoryCategory::Conversation, + MemoryCategory::Custom("core".into()), + MemoryCategory::Custom("tool_memory".into()), + ] { + assert_eq!( + category.to_string().parse::().unwrap(), + category + ); + let json = serde_json::to_string(&category).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + category + ); + } + assert_eq!( + "project_notes".parse::().unwrap(), + MemoryCategory::Custom("project_notes".into()) + ); +} + +#[test] +fn memory_entry_roundtrip_preserves_optional_fields() { + let entry = MemoryEntry { + id: "id-1".into(), + key: "favorite_language".into(), + content: "Rust".into(), + namespace: Some("global".into()), + category: MemoryCategory::Core, + timestamp: "2026-02-16T00:00:00Z".into(), + session_id: Some("session-abc".into()), + score: Some(0.98), + taint: MemoryTaint::Internal, + }; + let json = serde_json::to_string(&entry).unwrap(); + let parsed: MemoryEntry = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.id, "id-1"); + assert_eq!(parsed.namespace.as_deref(), Some("global")); + assert_eq!(parsed.category, MemoryCategory::Core); + assert_eq!(parsed.session_id.as_deref(), Some("session-abc")); + assert_eq!(parsed.score, Some(0.98)); + assert_eq!(parsed.taint, MemoryTaint::Internal); +} + +#[test] +fn memory_taint_defaults_to_internal_for_legacy_rows() { + let legacy = r#"{ + "id":"x","key":"k","content":"c","namespace":null, + "category":"core","timestamp":"2026-01-01T00:00:00Z", + "session_id":null,"score":null + }"#; + let parsed: MemoryEntry = serde_json::from_str(legacy).unwrap(); + assert_eq!(parsed.taint, MemoryTaint::Internal); +} + +#[test] +fn memory_taint_db_str_roundtrip_and_fails_closed() { + assert_eq!(MemoryTaint::Internal.as_db_str(), "internal"); + assert_eq!(MemoryTaint::ExternalSync.as_db_str(), "external_sync"); + assert_eq!(MemoryTaint::from_db_str("internal"), MemoryTaint::Internal); + assert_eq!( + MemoryTaint::from_db_str("external_sync"), + MemoryTaint::ExternalSync + ); + // Unknown / corrupt values fail closed to the restrictive variant. + assert_eq!(MemoryTaint::from_db_str(""), MemoryTaint::ExternalSync); + assert_eq!( + MemoryTaint::from_db_str("EXTERNAL_SYNC"), + MemoryTaint::ExternalSync + ); + assert_eq!( + MemoryTaint::from_db_str("future"), + MemoryTaint::ExternalSync + ); +} + +#[test] +fn memory_taint_serde_unknown_values_fail_closed() { + assert_eq!( + serde_json::from_str::("\"unexpected\"").unwrap(), + MemoryTaint::ExternalSync + ); + assert_eq!( + serde_json::to_string(&MemoryTaint::ExternalSync).unwrap(), + "\"external_sync\"" + ); +} + +#[test] +fn memory_item_kind_serde_uses_snake_case() { + assert_eq!( + serde_json::to_string(&MemoryItemKind::Document).unwrap(), + "\"document\"" + ); + let decoded: MemoryItemKind = serde_json::from_str("\"episodic\"").unwrap(); + assert_eq!(decoded, MemoryItemKind::Episodic); +} + +#[test] +fn namespace_document_input_defaults_optional_fields() { + let value = json!({ + "namespace": "global", "key": "note-1", "title": "Title", + "content": "Body", "source_type": "manual", "priority": "normal", + "metadata": {}, "category": "core" + }); + let parsed: NamespaceDocumentInput = serde_json::from_value(value).unwrap(); + assert!(parsed.tags.is_empty()); + assert!(parsed.session_id.is_none()); + assert!(parsed.document_id.is_none()); + assert_eq!(parsed.taint, MemoryTaint::Internal); +} + +#[test] +fn namespace_document_input_taint_roundtrips_external_sync() { + let input = NamespaceDocumentInput { + namespace: "skill-gmail".into(), + key: "thread-1".into(), + title: "Subject".into(), + content: "Body".into(), + source_type: "composio-sync".into(), + priority: "medium".into(), + tags: Vec::new(), + metadata: json!({}), + category: "core".into(), + session_id: None, + document_id: None, + taint: MemoryTaint::ExternalSync, + }; + let value = serde_json::to_value(&input).unwrap(); + assert_eq!( + value.get("taint").and_then(|v| v.as_str()), + Some("external_sync") + ); + let parsed: NamespaceDocumentInput = serde_json::from_value(value).unwrap(); + assert_eq!(parsed.taint, MemoryTaint::ExternalSync); +} + +#[test] +fn retrieval_score_breakdown_default_is_zeroed() { + let b = RetrievalScoreBreakdown::default(); + assert_eq!(b.keyword_relevance, 0.0); + assert_eq!(b.vector_similarity, 0.0); + assert_eq!(b.graph_relevance, 0.0); + assert_eq!(b.episodic_relevance, 0.0); + assert_eq!(b.freshness, 0.0); + assert_eq!(b.final_score, 0.0); +} + +#[test] +fn memory_kv_record_roundtrips_with_optional_namespace() { + for record in [ + MemoryKvRecord { + namespace: None, + key: "theme".into(), + value: json!("dark"), + updated_at: 1.5, + }, + MemoryKvRecord { + namespace: Some("project".into()), + key: "state".into(), + value: json!({"open": true}), + updated_at: 2.5, + }, + ] { + let value = serde_json::to_value(&record).unwrap(); + let decoded: MemoryKvRecord = serde_json::from_value(value).unwrap(); + assert_eq!(decoded.namespace, record.namespace); + assert_eq!(decoded.key, record.key); + assert_eq!(decoded.value, record.value); + assert_eq!(decoded.updated_at, record.updated_at); + } +} + +#[test] +fn namespace_memory_hit_defaults_optional_fields_and_taint() { + let hit: NamespaceMemoryHit = serde_json::from_value(json!({ + "id": "hit-1", "kind": "document", "namespace": "global", + "key": "note-1", "title": "Title", "content": "Body", + "category": "core", "source_type": "manual", "updated_at": 3.5, + "score": 0.8, + "score_breakdown": { + "keyword_relevance": 0.5, "vector_similarity": 0.2, + "graph_relevance": 0.0, "episodic_relevance": 0.0, + "freshness": 0.1, "final_score": 0.8 + } + })) + .unwrap(); + assert!(hit.document_id.is_none()); + assert!(hit.chunk_id.is_none()); + assert!(hit.supporting_relations.is_empty()); + assert_eq!(hit.kind, MemoryItemKind::Document); + assert_eq!(hit.taint, MemoryTaint::Internal); +} diff --git a/src/openhuman/memory/api/version.rs b/src/openhuman/memory/api/version.rs new file mode 100644 index 0000000000..b160bb3c35 --- /dev/null +++ b/src/openhuman/memory/api/version.rs @@ -0,0 +1,91 @@ +//! The memory contract version and the compatibility rule that governs it. +//! +//! Re-exported at the crate root, so the canonical paths are +//! [`crate::openhuman::memory::api::CONTRACT_VERSION`] and [`crate::openhuman::memory::api::is_compatible`]. +//! +//! ## The rule +//! +//! `CONTRACT_VERSION` is `(major, minor)`: +//! +//! - **Minor bump — an addition that capability negotiation alone makes safe.** +//! A new [`crate::openhuman::memory::api::capabilities::Capability`] family is the canonical case: an +//! older driver simply never advertises it, the corresponding RPC methods are +//! unregistered, and the kernel never calls in. A new optional field on an +//! existing wire type, or a new error variant an older kernel can treat as +//! opaque, are the same shape — nothing that already compiled stops +//! compiling, and there is no way for an old driver to be asked for +//! something it never claimed to support. +//! - **Major bump — an existing signature changed, OR a method was added to an +//! already-advertised family.** A method's parameters or return type moved, a +//! mandatory family was added or removed, a wire string changed — or a driver +//! advertising an existing family (say [`crate::openhuman::memory::api::capabilities::Capability::Core`]) +//! now has to implement one more method on it. That last case looks additive +//! but is not: capability negotiation has **family granularity only** — there +//! is no way to advertise "`Core`, but without the new method" — so an older +//! driver that still advertises `Core` can be called into a method it does +//! not implement. Bump the major half instead, which forces every driver +//! claiming that family to actually implement the new surface before it can +//! bind again. +//! +//! ## Why only the major half gates the bind +//! +//! An out-of-process driver reports the version it speaks in its handshake +//! (`POST /v1/handshake` → `{ contract_version, driver_id, capabilities[] }`). +//! **A major mismatch refuses the bind**; a minor difference in either +//! direction is accepted, because capability negotiation already covers it: +//! +//! - remote minor > local minor — the driver advertises families this build has +//! never heard of. Unknown family strings are skipped during handshake +//! parsing, so this kernel simply never calls them. +//! - remote minor < local minor — the driver is missing families this build +//! knows about. It does not advertise them, so the corresponding RPC methods +//! are unregistered and the agent tools are absent. That is the ordinary +//! degradation path, not an error. +//! +//! Refusing on a minor difference would therefore reject a driver that is +//! perfectly usable, and would make adding a family a fleet-wide breaking +//! change — which is exactly what the major/minor split exists to avoid. +//! +//! Encoding the rule here rather than in prose means a caller cannot get it +//! subtly wrong: the bind path calls [`is_compatible`], never compares tuples +//! by hand. + +/// Version of the memory contract this crate defines, as `(major, minor)`. +/// +/// See the module docs for the bump rule. Bump the **minor** half only for an +/// addition capability negotiation alone makes safe — a new capability family, +/// a new optional wire field, a new opaque-to-old-kernels error variant. Bump +/// the **major** half — and reset the minor to `0` — for an existing signature +/// change, a mandatory family change, a wire string change, **or a new method +/// added to a family a driver may already advertise** (negotiation is +/// family-granular, not method-granular, so that case cannot be made minor-safe +/// by negotiation alone). +pub const CONTRACT_VERSION: (u16, u16) = (2, 0); + +/// Whether a driver speaking `remote` can be bound against this build. +/// +/// Compatible exactly when the major halves match. See the module docs for why +/// the minor half is informational. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::memory::api::{is_compatible, CONTRACT_VERSION}; +/// +/// // The version this build speaks is always compatible with itself. +/// assert!(is_compatible(CONTRACT_VERSION)); +/// +/// // A minor difference in either direction is fine — capability negotiation +/// // covers the delta. +/// assert!(is_compatible((CONTRACT_VERSION.0, CONTRACT_VERSION.1 + 7))); +/// +/// // A major mismatch refuses the bind. +/// assert!(!is_compatible((CONTRACT_VERSION.0 + 1, 0))); +/// ``` +pub fn is_compatible(remote: (u16, u16)) -> bool { + remote.0 == CONTRACT_VERSION.0 +} + +#[cfg(test)] +#[path = "version_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/version_tests.rs b/src/openhuman/memory/api/version_tests.rs new file mode 100644 index 0000000000..b6baf3935c --- /dev/null +++ b/src/openhuman/memory/api/version_tests.rs @@ -0,0 +1,83 @@ +//! Unit tests for the contract version rule in [`super`]. +//! +//! The rule these pin is the one from the kernel design: a **minor** bump means +//! a capability was added and stays compatible; a **major** mismatch refuses +//! the bind. + +use super::*; + +#[test] +fn contract_version_starts_at_one_zero() { + assert_eq!(CONTRACT_VERSION, (2, 0)); +} + +#[test] +fn own_version_is_compatible_with_itself() { + assert!(is_compatible(CONTRACT_VERSION)); +} + +#[test] +fn a_minor_bump_stays_compatible_in_both_directions() { + let (major, minor) = CONTRACT_VERSION; + + // Remote ahead: it advertises families this build does not know. Unknown + // family strings are skipped during handshake parsing. + assert!(is_compatible((major, minor + 1))); + assert!(is_compatible((major, minor + 25))); + assert!(is_compatible((major, u16::MAX))); + + // Remote behind: it lacks families this build knows. Those simply are not + // advertised, so the surface degrades — the ordinary path, not an error. + assert!(is_compatible((major, minor.saturating_sub(1)))); + assert!(is_compatible((major, 0))); +} + +#[test] +fn a_major_mismatch_refuses_the_bind() { + let (major, minor) = CONTRACT_VERSION; + + // Remote ahead by a major: an existing signature changed under us. + assert!(!is_compatible((major + 1, 0))); + assert!(!is_compatible((major + 1, minor))); + assert!(!is_compatible((major + 1, u16::MAX))); + + // Remote behind by a major: same reasoning, other direction. A newer minor + // does not rescue an older major. + assert!(!is_compatible((major - 1, u16::MAX))); + assert!(!is_compatible((0, 0))); +} + +#[test] +fn adding_a_method_to_an_already_advertised_family_requires_a_major_bump() { + // Capability negotiation has family granularity, not method granularity: + // there is no way to advertise "Core, but without the new method". So a + // method added to a family a driver may already advertise (e.g. Core, + // Recall) cannot be made minor-safe by negotiation the way a brand-new + // capability family can — an older driver still advertising that family + // would be called into a method it never implemented. This is why the + // module docs classify that addition as a MAJOR bump, not minor, even + // though it looks additive. This test exists so the rule cannot be + // re-derived from `is_compatible`'s code alone, which only encodes "major + // halves must match" and says nothing about *why* a same-family method + // addition belongs on the major side of that line. + assert!( + !is_compatible((CONTRACT_VERSION.0 + 1, 0)), + "a method added to an existing family must ship as a major bump, \ + which this asserts refuses the bind against an old build" + ); +} + +#[test] +fn compatibility_depends_only_on_the_major_half() { + let (major, _) = CONTRACT_VERSION; + for minor in [0u16, 1, 2, 7, 999, u16::MAX] { + assert!( + is_compatible((major, minor)), + "minor {minor} should not affect compatibility" + ); + assert!( + !is_compatible((major + 1, minor)), + "minor {minor} must not rescue a major mismatch" + ); + } +} diff --git a/src/openhuman/memory/api/wire.rs b/src/openhuman/memory/api/wire.rs new file mode 100644 index 0000000000..df6e701ac4 --- /dev/null +++ b/src/openhuman/memory/api/wire.rs @@ -0,0 +1,133 @@ +//! Error names for a driver reached over a wire, and the mapping both ends use. +//! +//! # Why this is here and not in the transport +//! +//! A driver can be in-process, in a loadable module, or behind a socket. The +//! last two need [`MemoryError`] to survive a round trip through a +//! `(name, message)` pair, because that is all a bus or an HTTP status gives +//! you. +//! +//! The mapping could have lived in whichever adapter needed it first. It lives +//! here instead because there will be more than one adapter, and two copies of +//! a name table drift: the module side starts answering +//! `…Error.PathEscape` while the host side still only recognises +//! `…Error.Invalid`, and the symptom is a security-relevant error +//! silently reclassified as a caller mistake. One table, used by both ends, with +//! [`round_trips_every_variant`](self) pinning it. +//! +//! # One name per variant, not one per outcome class +//! +//! An earlier sketch collapsed these onto three names — "the caller can fix it", +//! "the capability is absent", "something broke" — on the grounds that a host has +//! only those three responses. That is wrong for two reasons. +//! +//! A host does not merely *react* to a driver error; it **is** a +//! [`MemoryProvider`](crate::openhuman::memory::api::provider::MemoryProvider) to everything above it, +//! so it has to hand its own callers a `MemoryError`. Collapsing on the way out +//! and guessing on the way back in would turn a `NotFound` into an `Invalid`, +//! and `get`'s contract says a missing entry is `Ok(None)` while an `Invalid` is +//! a real failure — so the guess is observable. +//! +//! And `PathEscape` is not interchangeable with `Invalid`. It reports a symlink +//! or traversal attempt that left the workspace sandbox, which a host may want +//! to log, report or refuse to retry differently from a malformed argument. +//! +//! # Unrecognised names are backend failures +//! +//! [`from_wire`] maps anything it does not know to [`MemoryError::Other`], never +//! to [`MemoryError::Invalid`]. A driver newer than this build may name an error +//! this table has no variant for, and telling a caller its input was wrong when +//! it was not sends it into a rewrite loop over something already correct. +//! +//! # Messages, and what must not be in them +//! +//! The name is the contract; the message is for a human. Neither may carry a +//! namespace key, an entry's content, a recall query, a credential or an +//! absolute path — memory content is user data, and an error string is not a +//! place for it. `Io` and `Serde` are deliberately flattened into a message +//! here, because reconstructing a live `std::io::Error` or +//! `serde_json::Error` on the far side is not possible and not useful. + +use crate::openhuman::memory::api::error::MemoryError; + +/// A requested record, source or node was not found. +pub const NOT_FOUND: &str = "ai.tinyhumans.tinymemory.Error.NotFound"; +/// Caller-supplied input failed validation. +pub const INVALID: &str = "ai.tinyhumans.tinymemory.Error.Invalid"; +/// A configured budget was exceeded. +pub const BUDGET_EXCEEDED: &str = "ai.tinyhumans.tinymemory.Error.BudgetExceeded"; +/// A path escaped the workspace sandbox. +pub const PATH_ESCAPE: &str = "ai.tinyhumans.tinymemory.Error.PathEscape"; +/// An underlying IO failure. +pub const IO: &str = "ai.tinyhumans.tinymemory.Error.Io"; +/// A serialization or deserialization failure. +pub const SERDE: &str = "ai.tinyhumans.tinymemory.Error.Serde"; +/// The driver does not implement the named capability family. +pub const UNSUPPORTED: &str = "ai.tinyhumans.tinymemory.Error.Unsupported"; +/// An opaque lower-level failure. +pub const OTHER: &str = "ai.tinyhumans.tinymemory.Error.Other"; + +/// The wire name for `error`. +/// +/// Total by construction: the `match` is exhaustive, so a variant added to +/// [`MemoryError`] is a compile error here rather than a silent fallthrough onto +/// [`OTHER`]. +#[must_use] +pub fn wire_name(error: &MemoryError) -> &'static str { + match error { + MemoryError::NotFound(_) => NOT_FOUND, + MemoryError::Invalid(_) => INVALID, + MemoryError::BudgetExceeded(_) => BUDGET_EXCEEDED, + MemoryError::PathEscape(_) => PATH_ESCAPE, + MemoryError::Io(_) => IO, + MemoryError::Serde(_) => SERDE, + MemoryError::Unsupported { .. } => UNSUPPORTED, + MemoryError::Other(_) => OTHER, + } +} + +/// The message to send alongside [`wire_name`]. +/// +/// For most variants this is the inner string rather than the `Display` output, +/// so the receiving side can rebuild the variant without the prefix +/// (`"invalid input: "`, …) being baked into the payload twice. +#[must_use] +pub fn wire_message(error: &MemoryError) -> String { + match error { + MemoryError::NotFound(message) + | MemoryError::Invalid(message) + | MemoryError::BudgetExceeded(message) + | MemoryError::PathEscape(message) => message.clone(), + MemoryError::Unsupported { capability } => capability.clone(), + // No inner string to lift: these carry a foreign error type, so the + // rendered form is all there is. + MemoryError::Io(inner) => inner.to_string(), + MemoryError::Serde(inner) => inner.to_string(), + MemoryError::Other(inner) => inner.to_string(), + } +} + +/// Rebuild a [`MemoryError`] from a `(name, message)` pair. +/// +/// An unrecognised `name` becomes [`MemoryError::Other`] — see the module docs +/// on why it must not become [`MemoryError::Invalid`]. +#[must_use] +pub fn from_wire(name: &str, message: &str) -> MemoryError { + match name { + NOT_FOUND => MemoryError::NotFound(message.to_string()), + INVALID => MemoryError::Invalid(message.to_string()), + BUDGET_EXCEEDED => MemoryError::BudgetExceeded(message.to_string()), + PATH_ESCAPE => MemoryError::PathEscape(message.to_string()), + // `std::io::Error` cannot be reconstructed with its original kind from a + // string, and inventing one would be worse than being honest that this + // crossed a wire. The message is preserved. + IO => MemoryError::Other(anyhow::anyhow!("io error: {message}")), + SERDE => MemoryError::Other(anyhow::anyhow!("serde error: {message}")), + UNSUPPORTED => MemoryError::unsupported_raw(message), + _ => MemoryError::Other(anyhow::anyhow!("{message}")), + } +} + +#[cfg(test)] +#[path = "wire_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/api/wire_tests.rs b/src/openhuman/memory/api/wire_tests.rs new file mode 100644 index 0000000000..7fdba08c13 --- /dev/null +++ b/src/openhuman/memory/api/wire_tests.rs @@ -0,0 +1,104 @@ +//! The name table is a contract, so these tests pin it rather than exercise it. + +use super::{from_wire, wire_message, wire_name}; +use crate::openhuman::memory::api::capabilities::Capability; +use crate::openhuman::memory::api::error::MemoryError; + +/// Every variant, so a new one fails to compile in `wire_name` and fails here. +fn every_variant() -> Vec { + vec![ + MemoryError::NotFound("thread-7".to_string()), + MemoryError::Invalid("limit must be positive".to_string()), + MemoryError::BudgetExceeded("depth 12 exceeds 8".to_string()), + MemoryError::PathEscape("symlink leaves workspace".to_string()), + MemoryError::Io(std::io::Error::other("disk gone")), + MemoryError::Serde(serde_json::from_str::("nope").unwrap_err()), + MemoryError::unsupported(Capability::Tree), + MemoryError::Other(anyhow::anyhow!("engine stopped")), + ] +} + +#[test] +fn round_trips_every_variant() { + for error in every_variant() { + let name = wire_name(&error); + let message = wire_message(&error); + let rebuilt = from_wire(name, &message); + + // Io and Serde deliberately degrade to `Other`: neither foreign error + // type can be reconstructed from a string. Everything else must come + // back as the same variant, because a host re-raises it to its own + // callers and the variant is what they match on. + match (&error, &rebuilt) { + (MemoryError::Io(_) | MemoryError::Serde(_), MemoryError::Other(_)) => {} + _ => assert_eq!( + std::mem::discriminant(&error), + std::mem::discriminant(&rebuilt), + "{name} did not round-trip to the same variant" + ), + } + assert!( + rebuilt.to_string().contains(message.trim()) || message.is_empty(), + "{name} lost its message: {rebuilt}" + ); + } +} + +#[test] +fn every_name_is_distinct() { + let mut names: Vec<&str> = every_variant().iter().map(wire_name).collect(); + let before = names.len(); + names.sort_unstable(); + names.dedup(); + assert_eq!(before, names.len(), "two variants share a wire name"); +} + +#[test] +fn an_unrecognised_name_is_a_backend_failure_not_an_input_error() { + // The load-bearing case. A driver newer than this build names something we + // have no variant for; classifying it as `Invalid` would tell a caller its + // request was wrong and send it into a rewrite loop. + let rebuilt = from_wire("ai.tinyhumans.tinymemory.Error.SomethingNewer", "hmm"); + assert!(matches!(rebuilt, MemoryError::Other(_)), "{rebuilt:?}"); +} + +#[test] +fn a_path_escape_does_not_collapse_onto_invalid() { + // These were nearly given one shared name. A sandbox escape is not a + // malformed argument, and a host may log or refuse to retry it differently. + assert_ne!( + wire_name(&MemoryError::PathEscape("x".to_string())), + wire_name(&MemoryError::Invalid("x".to_string())) + ); +} + +#[test] +fn a_missing_entry_stays_not_found() { + // `get`'s contract makes a missing entry `Ok(None)` and an `Invalid` a real + // failure, so conflating the two is observable to a caller. + let rebuilt = from_wire(super::NOT_FOUND, "absent"); + assert!(matches!(rebuilt, MemoryError::NotFound(_)), "{rebuilt:?}"); +} + +#[test] +fn an_unsupported_capability_keeps_its_family_name() { + let error = MemoryError::unsupported(Capability::Diff); + let rebuilt = from_wire(wire_name(&error), &wire_message(&error)); + match rebuilt { + MemoryError::Unsupported { capability } => { + assert_eq!(capability, Capability::Diff.as_str()); + } + other => panic!("expected Unsupported, got {other:?}"), + } +} + +#[test] +fn an_unknown_capability_name_off_the_wire_survives() { + // A driver on a newer minor contract may name a family this build has no + // `Capability` for. It must not be dropped or fail to parse. + let rebuilt = from_wire(super::UNSUPPORTED, "vendor_extension"); + match rebuilt { + MemoryError::Unsupported { capability } => assert_eq!(capability, "vendor_extension"), + other => panic!("expected Unsupported, got {other:?}"), + } +} diff --git a/src/openhuman/memory/binding.rs b/src/openhuman/memory/binding.rs index 41962e0941..dbcbc2fb77 100644 --- a/src/openhuman/memory/binding.rs +++ b/src/openhuman/memory/binding.rs @@ -10,7 +10,7 @@ //! which keys on the context's workspace dir. The cache below is deliberately //! shaped like //! [`memory::people::store::for_workspace`](crate::openhuman::memory::people::store::for_workspace) -//! — a **workspace-and-config-keyed map** — and deliberately *not* like +//! — a **workspace-keyed map** — and deliberately *not* like //! [`memory::global`](crate::openhuman::memory::global), which is a single slot //! holding "the one active-user workspace". //! @@ -25,7 +25,7 @@ //! //! ## Two vocabularies meet here, on purpose //! -//! [`tinycortex_api`] is the *memory contract*: `MemoryProvider`, +//! [`crate::openhuman::memory::api`] is the host-owned memory contract: `MemoryProvider`, //! `Capabilities`, `MemoryHealth`. [`crate::core::subsystem`] is the kernel's //! *generic* driver vocabulary shared with the subsystems that come after //! memory: `DriverClass`, `DriverCapabilities`, `DriverHealth`, `BoundDriver`. @@ -34,67 +34,46 @@ //! redefined here precisely because it is a *host* fact about how a driver was //! bound, identical for every subsystem. //! -//! ## Scope of this step (M3d) -//! -//! The [`DriverClass::Embedded`] arm of [`build`] binds the real -//! [`EmbeddedMemoryProvider`], which wraps the in-process tinycortex engine. -//! [`DriverClass::Null`] still binds [`NullMemoryProvider`] — an operator who -//! wrote `driver = "null"` asked for `/dev/null` and must get it — and so does -//! every fallback. -//! -//! The embedded driver now implements **all thirteen** families, so a bound -//! context and an unbound one advertise the same set. That was the whole point -//! of M3: before it, binding *narrowed* the advertised set from thirteen -//! families to the null placeholder's three, which made gating anything on -//! `memory_capabilities()` actively dangerous. It is now safe, and M4 is where -//! that gating lands. -//! -//! A fallback binding still advertises only the mandatory three, because a -//! fallback really is the null placeholder — that is the honest answer, not a -//! leftover. +//! The built-in driver is the compiled TinyMemory TinyBus module. The host no +//! longer exposes an embedded engine class for memory. use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock, RwLock}; -use tinycortex_api::capabilities::Capabilities; -use tinycortex_api::health::MemoryHealth; -use tinycortex_api::null::{NullMemoryProvider, NULL_DRIVER_ID}; -use tinycortex_api::provider::MemoryProvider; -use tinycortex_api::CONTRACT_VERSION; - -use tinymemory::registry::{ - ConfigLabels, DriverClass as ContractDriverClass, DriverEntry, DriverRegistry, -}; +use crate::openhuman::memory::api::capabilities::Capabilities; +use crate::openhuman::memory::api::health::MemoryHealth; +use crate::openhuman::memory::api::null::{NullMemoryProvider, NULL_DRIVER_ID}; +use crate::openhuman::memory::api::provider::MemoryProvider; +use crate::openhuman::memory::api::CONTRACT_VERSION; +use crate::openhuman::memory::guard::{GuardPolicy, MemoryGuard}; use crate::core::subsystem::{ BoundDriver, DriverCapabilities, DriverClass, DriverHealth, SubsystemSlot, }; -use crate::openhuman::memory::driver::embedded::EmbeddedMemoryProvider; -use crate::openhuman::memory::guard::{GuardPolicy, MemoryGuard}; -use tinymemory_api::host::MemoryHooksConfig; -use tinymemory_api::host::MemorySubsystemConfig; +use crate::openhuman::config::schema::MemorySubsystemConfig; + +/// Registry id of the built-in TinyMemory module. +pub(crate) const MODULE_ID: &str = "tinymemory"; /// Why a bind fell back to the placeholder driver. /// -/// Defined in [`tinymemory::registry`] alongside the admission rules that -/// produce it. `reason` is operator-facing: it is logged, published on the -/// event bus, and rendered in status, so it must never interpolate -/// `credential_ref` or `endpoint` from -/// [`tinymemory_api::host::MemoryDriverConfig`], which carries a -/// manual redacting `Debug` for exactly that reason. The crate enforces this -/// structurally — [`DriverEntry`] carries neither field, so a refusal built -/// there cannot reach one. Pinned by +/// `reason` is operator-facing: it is logged, published on the event bus, and +/// rendered in status. It must therefore never interpolate `credential_ref` or +/// `endpoint` from [`crate::openhuman::config::schema::MemoryDriverConfig`], +/// which carries a manual redacting `Debug` for exactly that reason. Pinned by /// `fallback_reason_never_contains_credential_ref_or_endpoint`. -pub use tinymemory::registry::FallbackReason; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FallbackReason { + /// The driver id that was asked for in `[subsystems.memory] driver`. + pub configured_driver: String, + /// Why it was refused. + pub reason: String, +} /// One bound memory driver, for one workspace. pub struct MemoryBinding { provider: Arc, - /// The policy decorator over [`Self::unguarded_provider`] — the handle product code - /// receives, via `CoreContext::memory()`. Built here rather than by each - /// caller so "every caller gets a guarded handle" holds by construction, - /// the same way `capabilities()` is asked exactly once by construction. guard: Arc, driver_id: String, class: DriverClass, @@ -107,35 +86,23 @@ pub struct MemoryBinding { } impl MemoryBinding { - /// The bound driver, **unguarded**. - /// - /// Retained for identity/health/status, which are liveness probes rather - /// than product code (`memory::ops::provider` is the one production - /// caller). New call sites want [`Self::guard`] — see - /// `CoreContext::memory()`. - /// - /// Named `unguarded_provider` rather than `provider` on purpose. The - /// enforcement lint in `memory::bypass_allowlist_tests` matches text, and a - /// `.provider(` needle would over-match `TaskSourceFilter::provider()` and - /// `ModelRef::provider()` — six junk allowlist entries, which is exactly - /// the rot `bypass_allowlist_has_no_stale_entries` exists to prevent. A - /// distinctive name gives the lint a needle with no false positives, and - /// puts the hazard in the reader's face at the call site. - /// - /// Visibility is narrowed to the memory family so the lint's text match is - /// backed by a *compiler*-enforced boundary: even if `MemoryBinding` grows - /// another reachable path, no module outside `openhuman::memory` can name - /// this accessor at all. + /// The bound driver. + pub fn provider(&self) -> &Arc { + &self.provider + } + pub(crate) fn unguarded_provider(&self) -> &Arc { &self.provider } - /// The guarded driver — the only handle product code should hold - /// (`docs/specs/kernel.md` §3.4). pub fn guard(&self) -> Arc { Arc::clone(&self.guard) } + pub fn disables_memory(&self) -> bool { + self.class == DriverClass::Null && self.fallback.is_none() + } + /// The id of the driver that actually bound — `"null"` after a fallback, /// not the id that was asked for (that is in [`Self::fallback`]). pub fn driver_id(&self) -> &str { @@ -158,27 +125,6 @@ impl MemoryBinding { self.fallback.as_ref() } - /// Whether the operator asked for memory to be **off**. - /// - /// True only for a deliberate `[subsystems.memory] driver = "null"` — the - /// class alone is not enough, because a *fallback* also binds the null - /// placeholder and a misconfiguration must not silently take memory away - /// with it. A fallback is loud (`fallback()` is `Some`, status reports it) - /// and keeps the surface present. - /// - /// Read by [`CoreContext::memory_capabilities`](crate::core::runtime::context::CoreContext::memory_capabilities), - /// which answers with the empty set here, so the memory RPC methods and - /// memory agent tools are **absent** rather than present-and-answering off - /// some other store. That matters because most memory handlers still reach - /// the engine directly through `active_memory_client()` — the guarded - /// re-point is incremental and tracked in - /// `docs/specs/memory-guard-allowlist.md` — so leaving the surface - /// registered under a null binding would read the embedded SQLite store an - /// operator believed they had turned off. - pub fn disables_memory(&self) -> bool { - self.class == DriverClass::Null && self.fallback.is_none() - } - /// This binding in the kernel's generic vocabulary, for the subsystem /// registry and `subsystems_status` (kernel.md §6 item 6). This is the /// memory adapter `core::subsystem`'s module docs said would land later. @@ -223,83 +169,109 @@ pub fn unbound_default_capabilities() -> Capabilities { Capabilities::all() } -/// The registry of driver ids whose class this host fixes. -/// -/// [`DriverRegistry::builtin`] already reserves `null` and `tinycortex`, which -/// are exactly this host's two built-in ids — so the builtin set is used as-is -/// rather than re-declared. A host bundling an adapter the crate does not know -/// about would add it here with `with_reserved`. -fn registry() -> DriverRegistry { - DriverRegistry::builtin() -} - -/// The config-path spellings quoted back to the operator in refusal messages. -/// -/// The crate does not know what this host's config file looks like; these are -/// the blocks an operator would actually edit. -const CONFIG_LABELS: ConfigLabels<'static> = ConfigLabels { - section: "[subsystems.memory]", - drivers: "[subsystems.memory.drivers]", - driver_entry: "[subsystems.memory.drivers.]", -}; - -/// The class a built-in driver id is *fixed* to, or `None` for any other id. -/// -/// Both built-in ids name one specific implementation, so the registry is the -/// authority for their class in every path — the implicit one and the explicit -/// `class = …` line, which may only confirm what this returns. -pub(crate) fn reserved_class(id: &str) -> Option { - registry().reserved_class(id).map(from_contract_class) -} - -/// The contract's driver class in the kernel's generic vocabulary. -/// -/// A total match, which is why both enums were shaped one-for-one. -/// The kernel's own enum is deliberately not replaced by the contract's: it is -/// shared with the subsystems that come after memory, which must not inherit -/// their vocabulary from a *memory* crate. -fn from_contract_class(class: ContractDriverClass) -> DriverClass { - match class { - ContractDriverClass::Embedded => DriverClass::Embedded, - ContractDriverClass::External => DriverClass::External, - ContractDriverClass::Module => DriverClass::Module, - ContractDriverClass::Null => DriverClass::Null, - } -} - /// Decide, from config alone, whether the configured driver may bind. /// /// Pure — no I/O, no globals — so the fail-closed trust rule is unit-testable /// without booting anything. /// -/// The rules themselves live in [`tinymemory::registry`], because they are the -/// one part of binding with the same correct answer for every host: a built-in -/// id's class is fixed and an explicit `class` line may only confirm it, an -/// unknown id is refused rather than guessed, and an external driver is -/// fail-closed on trust. This function is the projection from *this* host's -/// config shape onto that decision, and the conversion back into the kernel's -/// generic driver vocabulary. -/// -/// Note what is deliberately **not** passed to the crate: only the `class` and -/// `trust_state` of the driver entry cross, never `credential_ref` or -/// `endpoint`. A refusal message is operator-facing and logged, so the narrow -/// projection is what makes "no secret can appear in a refusal" structural -/// rather than a rule someone has to remember. -/// /// # Errors /// /// Returns the [`FallbackReason`] to record and publish when the configured /// driver is refused. Callers fall back rather than failing: kernel.md §3.7 /// requires the subsystem stay bound, loudly. pub fn admit(cfg: &MemorySubsystemConfig) -> Result<(String, DriverClass), FallbackReason> { - let id = cfg.driver.trim(); - let entry = cfg.drivers.get(id).map(|entry| DriverEntry { - class: entry.class.as_deref(), - trust_state: entry.trust_state.as_str(), - }); - - let admission = registry().admit(&cfg.driver, entry, CONFIG_LABELS)?; - Ok((admission.id, from_contract_class(admission.class))) + let configured_id = cfg.driver.trim(); + if configured_id.is_empty() { + return Err(FallbackReason { + configured_driver: String::new(), + reason: "[subsystems.memory] driver is empty".to_string(), + }); + } + + let refuse = |reason: &str| FallbackReason { + configured_driver: configured_id.to_string(), + reason: reason.to_string(), + }; + + // Temporary persisted-config alias. The schema still comes from the + // legacy contract until its remaining engine callers are moved onto the + // host-owned copy; both values bind the compiled module and report its + // actual id. Remove this alias with that final schema cutover. + const LEGACY_MODULE_ID: &str = "tinycortex"; + let id = if configured_id == LEGACY_MODULE_ID { + MODULE_ID + } else { + configured_id + }; + + // The two built-ins need no `[subsystems.memory.drivers.]` entry. + let Some(entry) = cfg + .drivers + .get(configured_id) + .or_else(|| cfg.drivers.get(id)) + else { + return match id { + NULL_DRIVER_ID => Ok((id.to_string(), DriverClass::Null)), + MODULE_ID => Ok((id.to_string(), DriverClass::Module)), + _ => Err(refuse(&format!( + "driver '{id}' is not built in; add [subsystems.memory.drivers.{id}] with an explicit class line" + ))), + }; + }; + + let class = match entry.class.as_deref() { + None if id == NULL_DRIVER_ID => DriverClass::Null, + None if id == MODULE_ID => DriverClass::Module, + None => { + return Err(refuse(&format!( + "driver '{id}' is not built in and requires an explicit class line" + ))) + } + Some(raw) => DriverClass::parse(raw).map_err(|e| refuse(&e))?, + }; + + if class == DriverClass::Embedded { + return Err(refuse( + "embedded memory drivers are no longer supported; use the 'tinymemory' module driver", + )); + } + + let built_in_class = match id { + NULL_DRIVER_ID => Some(DriverClass::Null), + MODULE_ID => Some(DriverClass::Module), + _ => None, + }; + if let Some(expected) = built_in_class { + if class != expected { + return Err(refuse(&format!( + "built in driver '{configured_id}' has class '{expected}' and cannot be re-classed as '{class}'" + ))); + } + } + + if class == DriverClass::Module && id != MODULE_ID { + return Err(refuse(&format!( + "module driver '{id}' is not registered; the built-in memory module id is '{MODULE_ID}'" + ))); + } + + if class == DriverClass::External { + // kernel.md §3.4: fail-closed. Trust must be explicitly raised before + // an out-of-process driver is allowed to answer for memory. + if entry.trust_state != "trusted" { + return Err(refuse( + "external driver is untrusted: set trust_state = \"trusted\" \ + under [subsystems.memory.drivers] to allow this binding", + )); + } + // Distinct reason string from the trust refusal above, so the trust + // test cannot pass for the wrong reason. + return Err(refuse( + "external driver transport is not implemented yet (the http adapter lands in M4)", + )); + } + + Ok((id.to_string(), class)) } /// Build the binding for a workspace. Infallible by design: an inadmissible @@ -308,77 +280,13 @@ pub fn admit(cfg: &MemorySubsystemConfig) -> Result<(String, DriverClass), Fallb fn build(workspace_dir: &Path, cfg: &MemorySubsystemConfig) -> MemoryBinding { match admit(cfg) { Ok((driver_id, class)) => { - // Both the provider *and* the class it should be reported under. The - // two can differ: a build without the `modules` feature admits the - // module class but can only bind a placeholder, and reporting the - // admitted class there would advertise a module-backed driver with a - // null behind it — a live surface with no store, which is exactly - // what status output exists to make visible. - let (provider, class): (Arc, DriverClass) = match class { - // Construction is deliberately sync and I/O-free: this runs on - // `CoreContext::memory_binding`, which ~4000 pre-boot tests - // call with no tokio runtime. The driver resolves its client on - // first use — see `driver::embedded`'s module docs. - DriverClass::Embedded => ( - Arc::new(EmbeddedMemoryProvider::new(workspace_dir, cfg.hooks)), - DriverClass::Embedded, - ), - DriverClass::Null => (Arc::new(NullMemoryProvider::new()), DriverClass::Null), - // Unreachable: `admit` refuses every external driver above, so - // this arm cannot bind a transport that does not exist yet. - // Reported as `Null`, for the same reason as the arm below: what - // bound is a placeholder, and status must say so. - DriverClass::External => (Arc::new(NullMemoryProvider::new()), DriverClass::Null), - // This function builds an `Arc`, while `modules::memory::ModuleMemoryProvider` - // implements `tinymemory_api::provider::MemoryProvider`. Those are - // two different traits from two different crates. - // `TinyMemoryContractAdapter` is the bridge: it wraps the module - // provider and implements the tinycortex-side trait by converting - // each call across the seam (see its module docs for why most - // conversions destructure exhaustively while a few cross by serde - // round trip). It is a temporary bridge — it exists to be deleted - // once the binding itself migrates onto the tinymemory contract, - // at which point `ModuleMemoryProvider` binds directly here. - // - // Gated on the `modules` feature because the concrete provider - // type lives behind it (unlike the adapter above, which is - // generic and feature-independent). A build without `modules` - // cannot load the module driver at all, so it falls back to the - // same placeholder every other inadmissible driver gets, logged - // loudly rather than silently — kernel.md §3.7. - #[cfg(feature = "modules")] - DriverClass::Module => ( - Arc::new( - crate::openhuman::memory::driver::module_adapter::TinyMemoryContractAdapter::new( - Arc::new( - crate::openhuman::modules::memory::ModuleMemoryProvider::from_boot_policy(), - ), - ), - ), - DriverClass::Module, - ), - #[cfg(not(feature = "modules"))] - DriverClass::Module => { - log::warn!( - "[memory:binding] workspace={} configured driver='{}' resolves to the \ - module class, but this build was compiled without the `modules` \ - feature; falling back to the null placeholder", - workspace_dir.display(), - driver_id, - ); + let (provider, reported_class): (Arc, DriverClass) = + if class == DriverClass::Null { (Arc::new(NullMemoryProvider::new()), DriverClass::Null) - } - }; - // The configured trust state for the driver that actually bound. - // Absent `[subsystems.memory.drivers.]` entry ⇒ the fail-closed - // default, which only ever matters for an external class. - let trust_state = cfg - .drivers - .get(&driver_id) - .map(|entry| entry.trust_state.clone()) - .unwrap_or_else(|| crate::openhuman::memory::guard::policy::TRUSTED.to_string()); - let binding = bind_provider(provider, driver_id, class, cfg.hooks, trust_state, None); + } else { + module_provider(workspace_dir) + }; + let binding = bind_provider(provider, driver_id, reported_class, None); log::info!( "[memory:binding] workspace={} bound driver='{}' class={} capabilities=[{}]", workspace_dir.display(), @@ -403,8 +311,8 @@ fn build(workspace_dir: &Path, cfg: &MemorySubsystemConfig) -> MemoryBinding { ); // Sync, and a no-op when the bus is not yet initialized, so this is // safe to call pre-boot with no `#[cfg(test)]` guard. - crate::openhuman::memory::events::publish( - crate::openhuman::memory::events::MemoryEvent::DriverBindFailed { + crate::core::bus::BUS.publish( + crate::core::events::DomainEvent::MemoryDriverBindFailed { configured_driver: fallback.configured_driver.clone(), bound_driver: NULL_DRIVER_ID.to_string(), reason: fallback.reason.clone(), @@ -414,18 +322,55 @@ fn build(workspace_dir: &Path, cfg: &MemorySubsystemConfig) -> MemoryBinding { Arc::new(NullMemoryProvider::new()), NULL_DRIVER_ID.to_string(), DriverClass::Null, - cfg.hooks, - // The fallback binds the in-process placeholder, so there is no - // boundary to cross and nothing to trust-gate. The refused - // driver's own trust_state is deliberately NOT carried over — - // it describes a binding that did not happen. - crate::openhuman::memory::guard::policy::TRUSTED.to_string(), Some(fallback), ) } } } +#[cfg(all(feature = "modules", not(test)))] +fn module_provider(_workspace_dir: &Path) -> (Arc, DriverClass) { + ( + Arc::new(crate::openhuman::modules::memory::ModuleMemoryProvider::from_boot_policy()), + DriverClass::Module, + ) +} + +#[cfg(all(feature = "modules", test))] +fn module_provider(_workspace_dir: &Path) -> (Arc, DriverClass) { + // Unit tests do not run the full boot sequence that publishes the module + // policy. A native module is loaded once per process and therefore captures + // the first workspace it receives. Pin every test binding to the same + // workspace as the process-global test client so concurrent tests cannot + // win module initialization with an unrelated tempdir and split guarded + // writes from legacy read-back calls. + let workspace_dir = crate::openhuman::memory::ops::ensure_shared_memory_client(); + let mut config = crate::openhuman::config::Config::default(); + config.workspace_dir = workspace_dir.clone(); + config.modules.install_dir = Some(workspace_dir.join("modules").to_string_lossy().into_owned()); + if let Some(path) = std::env::var_os("TINYMEMORY_TEST_MODULE") { + config + .modules + .overrides + .push(crate::openhuman::config::schema::ModuleOverride { + id: MODULE_ID.to_string(), + path: path.to_string_lossy().into_owned(), + }); + } + ( + Arc::new(crate::openhuman::modules::memory::ModuleMemoryProvider::new(Arc::new(config))), + DriverClass::Module, + ) +} + +#[cfg(not(feature = "modules"))] +fn module_provider(_workspace_dir: &Path) -> (Arc, DriverClass) { + log::warn!( + "[memory:binding] the 'modules' feature is disabled; binding the null memory provider" + ); + (Arc::new(NullMemoryProvider::new()), DriverClass::Null) +} + /// The single place `capabilities()` is asked. Every construction path — real /// bind, fallback, and the test seam — goes through here, so the "asked once /// per bind" property holds by construction rather than by convention. @@ -433,20 +378,16 @@ fn bind_provider( provider: Arc, driver_id: String, class: DriverClass, - hooks: MemoryHooksConfig, - trust_state: String, fallback: Option, ) -> MemoryBinding { let capabilities = provider.capabilities(); - // Built on the same single path, so a binding can never exist without its - // guard and no caller has to remember to construct one. let guard = Arc::new(MemoryGuard::new( Arc::clone(&provider), Arc::new(GuardPolicy::new( driver_id.clone(), class, - hooks, - trust_state, + crate::openhuman::config::schema::MemoryHooksConfig::default(), + "trusted", )), )); MemoryBinding { @@ -469,33 +410,14 @@ pub(crate) fn bind_provider_for_test( class: DriverClass, ) -> MemoryBinding { let driver_id = provider.driver_id().to_string(); - bind_provider( - provider, - driver_id, - class, - MemoryHooksConfig::default(), - crate::openhuman::memory::guard::policy::TRUSTED.to_string(), - None, - ) + bind_provider(provider, driver_id, class, None) } /// Per-workspace binding cache. Same shape as /// `memory::people::store::STORES` — see the module docs for why this is a map /// and not a slot. -/// -/// Keyed on the **binding-relevant config as well as the path**, not the path -/// alone, because a config change for an already-bound workspace must produce a -/// fresh binding. `CoreContext::rebind_workspace` deliberately treats "same -/// workspace, changed `[subsystems.memory]`" as a real rebind — a changed -/// `driver` / `hooks` / `drivers` (trust) all feed `build`, so a path-only key -/// would keep serving the previous driver until restart. Carrying -/// `MemorySubsystemConfig` in the key (it derives `Hash`) means a changed -/// config hits a different slot and binds fresh, while a returned-to config -/// still resolves its original binding. type BindingCacheKey = (PathBuf, MemorySubsystemConfig); -type BindingCache = RwLock>>; - -static BINDINGS: OnceLock = OnceLock::new(); +static BINDINGS: OnceLock>>> = OnceLock::new(); /// The bound memory driver for `workspace_dir`, constructing it on first use. /// @@ -526,9 +448,8 @@ pub fn for_workspace( .write() .map_err(|e| format!("[memory:binding] cache write lock poisoned: {e}"))?; // Re-check under the write lock: a racing caller may have bound the same - // workspace (and config) while we were building. Reuse theirs so one - // workspace never has two live drivers for the same config (kernel.md §3.1) - // and `capabilities()` stays asked once. + // workspace while we were building. Reuse theirs so one workspace never has + // two live drivers (kernel.md §3.1) and `capabilities()` stays asked once. let entry = guard.entry(key).or_insert_with(|| Arc::clone(&binding)); Ok(Arc::clone(entry)) } diff --git a/src/openhuman/memory/binding_tests.rs b/src/openhuman/memory/binding_tests.rs index b0206f0b0a..b8d9949c35 100644 --- a/src/openhuman/memory/binding_tests.rs +++ b/src/openhuman/memory/binding_tests.rs @@ -17,24 +17,23 @@ use std::sync::Arc; // `binding.rs` reaches these through its own `use` statements; a sibling test // module only inherits its `pub` items, so they are named again here. use crate::core::subsystem::{DriverHealth, SubsystemSlot}; -use tinycortex_api::capabilities::Capabilities; -use tinycortex_api::health::MemoryHealth; -use tinycortex_api::null::{NullMemoryProvider, NULL_DRIVER_ID}; -use tinycortex_api::provider::MemoryProvider; -use tinycortex_api::CONTRACT_VERSION; - -// Imported here rather than re-exported from `binding.rs`: since admission -// moved to `tinymemory::registry`, the production module no longer names this -// constant and an import kept alive only for the tests would read as dead code. -use crate::openhuman::memory::driver::embedded::EMBEDDED_DRIVER_ID; - +use crate::openhuman::memory::api::capabilities::Capabilities; +use crate::openhuman::memory::api::health::MemoryHealth; +use crate::openhuman::memory::api::null::{NullMemoryProvider, NULL_DRIVER_ID}; +use crate::openhuman::memory::api::provider::MemoryProvider; +use crate::openhuman::memory::api::CONTRACT_VERSION; + +use crate::openhuman::memory::api::capabilities::Capability; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::provider::types::{ + ExportPage, ExportRecord, ImportOutcome, SourceScope, +}; +use crate::openhuman::memory::api::provider::{MemoryCore, MemoryPortability, MemoryRecall}; +use crate::openhuman::memory::api::recall::OwnedRecallOpts; +use crate::openhuman::memory::api::types::{ + MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, +}; use async_trait::async_trait; -use tinycortex_api::capabilities::Capability; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::{ExportPage, ExportRecord, ImportOutcome, SourceScope}; -use tinycortex_api::provider::{MemoryCore, MemoryPortability, MemoryRecall}; -use tinycortex_api::recall::OwnedRecallOpts; -use tinycortex_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; use tinymemory_api::host::MemoryDriverConfig; @@ -57,10 +56,10 @@ fn external_driver_cfg(trust_state: &str) -> MemorySubsystemConfig { } #[test] -fn admit_default_config_binds_embedded_tinycortex() { +fn admit_default_config_binds_module_tinymemory() { let (id, class) = admit(&MemorySubsystemConfig::default()).expect("default config admits"); - assert_eq!(id, "tinycortex"); - assert_eq!(class, DriverClass::Embedded); + assert_eq!(id, "tinymemory"); + assert_eq!(class, DriverClass::Module); } #[test] @@ -75,24 +74,24 @@ fn admit_null_driver_binds_null_class() { } #[test] -fn admit_typo_d_embedded_driver_id_gets_embedded_class() { +fn admit_builtin_module_driver_id_gets_module_class() { // Regression for the reviewer finding: before this, any non-null id without // a drivers entry — a typo like "tinycortx", or an external backend that // forgot its table — was silently classified Embedded. Only the two built-in // ids admit implicitly. let cfg = MemorySubsystemConfig { - driver: "tinycortex".into(), + driver: "tinymemory".into(), ..Default::default() }; - let (id, class) = admit(&cfg).expect("the embedded default id admits"); - assert_eq!(id, "tinycortex"); - assert_eq!(class, DriverClass::Embedded); + let (id, class) = admit(&cfg).expect("the module default id admits"); + assert_eq!(id, "tinymemory"); + assert_eq!(class, DriverClass::Module); } #[test] fn admit_refuses_an_unregistered_non_null_driver_id() { // A typo or an external backend with no `drivers.` entry must not - // silently run the embedded engine under an invented driver id. + // silently run the module under an invented driver id. let cfg = MemorySubsystemConfig { driver: "supermemory".into(), ..Default::default() @@ -142,10 +141,7 @@ fn admit_refuses_non_builtin_id_even_with_a_drivers_entry_that_says_no_class() { } #[test] -fn admit_accepts_an_explicit_embedded_class_for_a_registered_id() { - // A drivers entry that explicitly names the embedded class is a deliberate - // declaration — that id genuinely means the in-process engine. Explicit - // beats implicit. +fn admit_refuses_an_unregistered_module_id() { let mut cfg = MemorySubsystemConfig { driver: "custom-mem".into(), ..Default::default() @@ -153,13 +149,19 @@ fn admit_accepts_an_explicit_embedded_class_for_a_registered_id() { cfg.drivers.insert( "custom-mem".into(), MemoryDriverConfig { - class: Some("embedded".into()), + class: Some("module".into()), ..Default::default() }, ); - let (id, class) = admit(&cfg).expect("explicit embedded class admits"); - assert_eq!(id, "custom-mem"); - assert_eq!(class, DriverClass::Embedded); + let refusal = admit(&cfg).expect_err("only a registered TinyBus module may bind"); + assert!(refusal.reason.contains("not registered")); +} + +#[test] +fn admit_refuses_the_removed_embedded_class() { + let refusal = admit(&cfg_with_class("custom-mem", "embedded")) + .expect_err("the in-process memory engine was removed"); + assert!(refusal.reason.contains("no longer supported")); } #[test] @@ -241,6 +243,59 @@ fn for_workspace_caches_binding_per_workspace() { ); } +#[cfg(feature = "modules")] +#[tokio::test] +async fn unrelated_test_binding_cannot_capture_the_module_workspace() { + let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK + .lock() + .await; + crate::openhuman::memory::ops::ensure_shared_memory_client(); + let unrelated = tempfile::tempdir().expect("unrelated workspace"); + let binding = + for_workspace(unrelated.path(), &MemorySubsystemConfig::default()).expect("module binding"); + let guard = binding.guard(); + let documents = guard.as_documents().expect("documents capability"); + let namespace = format!("memory-binding-test-{}", uuid::Uuid::new_v4()); + let key = format!( + "shared{}", + &uuid::Uuid::new_v4().as_simple().to_string()[..12] + ); + + documents + .put_document( + crate::openhuman::memory::api::types::NamespaceDocumentInput { + namespace: namespace.clone(), + key: key.clone(), + title: "Shared test module workspace".into(), + content: "The module must share the process-global test store.".into(), + source_type: "doc".into(), + priority: "normal".into(), + tags: vec![], + metadata: serde_json::Value::Null, + category: "general".into(), + session_id: None, + document_id: None, + taint: MemoryTaint::Internal, + }, + ) + .await + .expect("module-backed put"); + + let client = crate::openhuman::memory::global::client().expect("shared test client"); + let raw = client + .list_documents(Some(&namespace)) + .await + .expect("raw list"); + assert!( + raw["documents"] + .as_array() + .expect("documents array") + .iter() + .any(|document| document["key"] == key), + "an unrelated binding must not split the native module from the shared test store" + ); +} + #[test] fn same_workspace_with_changed_config_binds_fresh() { // `CoreContext::rebind_workspace` treats "same workspace, changed @@ -255,11 +310,11 @@ fn same_workspace_with_changed_config_binds_fresh() { ..Default::default() }; - let tiny = for_workspace(dir.path(), &default).expect("bind tinycortex"); - assert_eq!(tiny.driver_id(), "tinycortex"); + let tiny = for_workspace(dir.path(), &default).expect("bind tinymemory"); + assert_eq!(tiny.driver_id(), "tinymemory"); // Same (workspace, config) pair reuses the cached binding... - let tiny_again = for_workspace(dir.path(), &default).expect("re-bind tinycortex"); + let tiny_again = for_workspace(dir.path(), &default).expect("re-bind tinymemory"); assert!( Arc::ptr_eq(&tiny, &tiny_again), "unchanged config must reuse the cached binding" @@ -269,7 +324,7 @@ fn same_workspace_with_changed_config_binds_fresh() { let null_binding = for_workspace(dir.path(), &null).expect("bind null"); assert!( !Arc::ptr_eq(&tiny, &null_binding), - "changed config must bind fresh, not serve the stale tinycortex driver" + "changed config must bind fresh, not serve the stale tinymemory driver" ); assert_eq!(null_binding.driver_id(), "null"); @@ -277,7 +332,7 @@ fn same_workspace_with_changed_config_binds_fresh() { // the transient-mismatch half: a stale (workspace, config) pairing never // shadows the correct pair, so it cannot permanently pin a workspace to the // wrong driver (the atomicity concern in the login/logout rebind). - let tiny_reverted = for_workspace(dir.path(), &default).expect("re-bind tinycortex"); + let tiny_reverted = for_workspace(dir.path(), &default).expect("re-bind tinymemory"); assert!( Arc::ptr_eq(&tiny, &tiny_reverted), "returning to the original config must serve the original binding" @@ -285,16 +340,16 @@ fn same_workspace_with_changed_config_binds_fresh() { } #[test] -fn embedded_class_binds_the_embedded_driver_not_null() { +fn module_class_binds_the_module_driver_not_null() { // Plain `#[test]`: no tokio runtime. Binding must stay synchronous and - // I/O-free, which is why the embedded driver resolves its client lazily. + // I/O-free, which is why the module provider resolves its client lazily. let dir = tempfile::tempdir().unwrap(); let workspace = dir.path().join("never-created"); let binding = for_workspace(&workspace, &MemorySubsystemConfig::default()).expect("default bind"); - assert_eq!(binding.driver_id(), "tinycortex"); - assert_eq!(binding.class(), DriverClass::Embedded); + assert_eq!(binding.driver_id(), "tinymemory"); + assert_eq!(binding.class(), DriverClass::Module); assert!(binding.fallback().is_none()); assert_ne!(binding.unguarded_provider().driver_id(), NULL_DRIVER_ID); assert!(binding.capabilities().contains(Capability::Core)); @@ -306,7 +361,7 @@ fn embedded_class_binds_the_embedded_driver_not_null() { } #[test] -fn embedded_binding_advertises_every_family() { +fn module_binding_advertises_every_family() { // Widened once per M3 step; M3d is the last one. The interesting assertion // is the second: a *bound* context and an *unbound* one now agree, which // they did not for the whole of M2/M3a-c. @@ -509,7 +564,7 @@ impl MemoryProvider for CountingProvider { #[test] fn capabilities_are_asked_exactly_once_per_bind() { let provider = Arc::new(CountingProvider::new()); - let binding = bind_provider_for_test(provider.clone(), DriverClass::Embedded); + let binding = bind_provider_for_test(provider.clone(), DriverClass::Module); for _ in 0..5 { assert_eq!(binding.capabilities(), Capabilities::all()); @@ -527,9 +582,9 @@ fn capabilities_are_asked_exactly_once_per_bind() { // --------------------------------------------------------------------------- // // A per-driver table may confirm a built-in id's class but never override it. -// Without that rule `driver = "null"` plus `class = "embedded"` builds the real +// Without that rule `driver = "null"` plus `class = "module"` builds the real // engine and persists memory under the id documented as `/dev/null`, and the -// inverse labels a store-nothing provider `tinycortex`. +// inverse labels a store-nothing provider `tinymemory`. fn cfg_with_class(driver: &str, class: &str) -> MemorySubsystemConfig { let mut cfg = MemorySubsystemConfig { @@ -547,9 +602,9 @@ fn cfg_with_class(driver: &str, class: &str) -> MemorySubsystemConfig { } #[test] -fn admit_refuses_an_embedded_class_override_on_the_null_driver() { - let refusal = admit(&cfg_with_class("null", "embedded")) - .expect_err("null must not be re-classed as embedded"); +fn admit_refuses_a_module_class_override_on_the_null_driver() { + let refusal = admit(&cfg_with_class("null", "module")) + .expect_err("null must not be re-classed as module"); assert_eq!(refusal.configured_driver, "null"); assert!( refusal.reason.contains("built in"), @@ -559,10 +614,10 @@ fn admit_refuses_an_embedded_class_override_on_the_null_driver() { } #[test] -fn admit_refuses_a_null_class_override_on_the_embedded_driver() { - let refusal = admit(&cfg_with_class(EMBEDDED_DRIVER_ID, "null")) - .expect_err("tinycortex must not be re-classed as null"); - assert_eq!(refusal.configured_driver, EMBEDDED_DRIVER_ID); +fn admit_refuses_a_null_class_override_on_the_module_driver() { + let refusal = admit(&cfg_with_class(MODULE_ID, "null")) + .expect_err("tinymemory must not be re-classed as null"); + assert_eq!(refusal.configured_driver, MODULE_ID); assert!( refusal.reason.contains("built in"), "refusal must say the id is built in: {}", @@ -577,18 +632,17 @@ fn admit_accepts_a_class_line_that_agrees_with_the_built_in_id() { assert_eq!(id, "null"); assert_eq!(class, DriverClass::Null); - let (id, class) = - admit(&cfg_with_class(EMBEDDED_DRIVER_ID, "embedded")).expect("agreeing class admits"); - assert_eq!(id, EMBEDDED_DRIVER_ID); - assert_eq!(class, DriverClass::Embedded); + let (id, class) = admit(&cfg_with_class(MODULE_ID, "module")).expect("agreeing class admits"); + assert_eq!(id, MODULE_ID); + assert_eq!(class, DriverClass::Module); } #[test] -fn a_null_class_override_cannot_smuggle_the_embedded_engine_into_the_binding() { +fn a_null_class_override_cannot_smuggle_the_module_into_the_binding() { // The end-to-end shape of the refusal: `build` must not hand back an - // embedded provider for `driver = "null"`. + // module provider for `driver = "null"`. let dir = tempfile::tempdir().unwrap(); - let binding = for_workspace(dir.path(), &cfg_with_class("null", "embedded")).expect("binds"); + let binding = for_workspace(dir.path(), &cfg_with_class("null", "module")).expect("binds"); assert_eq!(binding.class(), DriverClass::Null); assert_eq!(binding.driver_id(), NULL_DRIVER_ID); @@ -631,7 +685,7 @@ fn a_fallback_to_null_does_not_disable_memory() { } #[test] -fn the_embedded_driver_never_disables_memory() { +fn the_module_driver_never_disables_memory() { let dir = tempfile::tempdir().unwrap(); let binding = for_workspace(dir.path(), &MemorySubsystemConfig::default()).expect("binds"); assert!(!binding.disables_memory()); diff --git a/src/openhuman/memory/bypass_allowlist_tests.rs b/src/openhuman/memory/bypass_allowlist_tests.rs index 2a9f895af6..7a9641a51b 100644 --- a/src/openhuman/memory/bypass_allowlist_tests.rs +++ b/src/openhuman/memory/bypass_allowlist_tests.rs @@ -105,10 +105,6 @@ const BYPASS_PATTERNS: &[(&str, &str)] = &[ ".get_document(", "pub(crate) read-one escape hatch, driver-only by contract", ), - ( - "EmbeddedMemoryProvider::new(", - "direct driver construction — must go through binding::for_workspace", - ), ( "NullMemoryProvider::new(", "direct driver construction — must go through binding::for_workspace", @@ -229,11 +225,6 @@ const ALLOWED: &[(&str, &str, &str)] = &[ "active_memory_client(", "carries a #[cfg(test)] memory_override seam the guard would bypass", ), - ( - "src/openhuman/flows/ops.rs", - "active_memory_client(", - "clear_namespace has no contract method; plus a memory_client_override test seam", - ), ( "src/openhuman/flows/tinyflows/memory_adapter.rs", ".memory_handle(", @@ -256,31 +247,11 @@ const ALLOWED: &[(&str, &str, &str)] = &[ "passes &MemoryClientRef into user_scopes::save; the contract has no such shape", ), // ── The driver and the binding: guarding these would be a cycle ── - ( - "src/openhuman/memory/binding.rs", - "EmbeddedMemoryProvider::new(", - "this is the construction path the lint protects", - ), ( "src/openhuman/memory/binding.rs", "NullMemoryProvider::new(", "this is the construction path the lint protects (fail-closed fallback)", ), - ( - "src/openhuman/memory/driver/embedded/documents.rs", - ".get_document(", - "this IS the driver — the escape hatch exists for exactly this call", - ), - ( - "src/openhuman/memory/driver/embedded/mod.rs", - ".memory_handle(", - "this IS the driver; it owns the engine handle by definition", - ), - ( - "src/openhuman/memory/driver/embedded/mod.rs", - "EmbeddedMemoryProvider::new(", - "the driver's own constructor", - ), ( "vendor/tinymemory/core/src/global.rs", "MemoryClient::from_workspace_dir(", @@ -312,16 +283,6 @@ const ALLOWED: &[(&str, &str, &str)] = &[ "global::client_if_ready(", "same definition site", ), - ( - "src/openhuman/memory/ops/kv_graph.rs", - "active_memory_client(", - "kv_get/kv_delete/graph_* have no contract twin or a lossy conversion", - ), - ( - "src/openhuman/memory/ops/learn.rs", - "active_memory_client(", - "list_namespaces() -> Vec vs the contract's Vec", - ), ( "src/openhuman/memory/ops/learn.rs", "global::client(", @@ -352,16 +313,6 @@ const ALLOWED: &[(&str, &str, &str)] = &[ "global::client(", "inline #[cfg(test)] module only; the scanner does not brace-track test blocks", ), - ( - "src/openhuman/memory/ops/tool_memory.rs", - ".memory_handle(", - "open_store() still serves the four handlers with no contract twin", - ), - ( - "src/openhuman/memory/ops/tool_memory.rs", - "active_memory_client(", - "tool_rule_put/get/*_json/*_for_prompt have no contract equivalent", - ), ( "vendor/tinymemory/core/src/store/client.rs", ".profile_conn(", diff --git a/src/openhuman/memory/driver/embedded/core_family.rs b/src/openhuman/memory/driver/embedded/core_family.rs deleted file mode 100644 index 4e574a5516..0000000000 --- a/src/openhuman/memory/driver/embedded/core_family.rs +++ /dev/null @@ -1,125 +0,0 @@ -//! [`MemoryCore`] for the embedded driver — store / get / forget / list / -//! namespaces. -//! -//! Four of the five are a straight delegation to the engine's [`Memory`] trait. -//! Two things are *not* straight, and both are load-bearing: -//! -//! 1. **`store` maps onto [`Memory::store_with_taint`], never [`Memory::store`].** -//! The contract has a single `store` that always carries a -//! [`MemoryTaint`](tinycortex_api::types::MemoryTaint), because provenance is -//! stamped by the host policy guard *before* the call. `Memory::store` hard-codes -//! `MemoryTaint::Internal`, so routing through it would launder -//! externally-sourced content into internal-trust content — the single -//! failure mode the guard exists to prevent. (`Memory::store_with_taint`'s -//! *trait default* also silently drops the taint; `UnifiedMemory` overrides -//! it, which is why this delegation is correct and the other is not.) -//! -//! 2. **`list(None, ..)` spans every namespace.** The contract says all-`None` -//! lists everything the driver holds; the engine's `list` normalises a -//! `None` namespace to `GLOBAL_NAMESPACE`, so a naive delegation would -//! silently return one namespace and call it "everything". The driver -//! composes `namespace_summaries()` with a per-namespace `list` instead — -//! two existing calls, no new query logic. - -use async_trait::async_trait; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::MemoryCore; -use tinycortex_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; - -use super::{engine_error, EmbeddedMemoryProvider}; - -#[async_trait] -impl MemoryCore for EmbeddedMemoryProvider { - async fn store( - &self, - namespace: &str, - key: &str, - content: &str, - category: MemoryCategory, - session_id: Option<&str>, - taint: MemoryTaint, - ) -> Result<(), MemoryError> { - log::debug!( - "[memory:driver:embedded] store namespace={namespace} key_len={} content_len={} \ - category={category} session={} taint={}", - key.len(), - content.len(), - session_id.unwrap_or("-"), - taint.as_db_str() - ); - // `store_with_taint`, never `store` — see the module docs. - self.memory() - .await? - .store_with_taint(namespace, key, content, category, session_id, taint) - .await - .map_err(engine_error) - } - - async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError> { - self.memory() - .await? - .get(namespace, key) - .await - .map_err(engine_error) - } - - async fn forget(&self, namespace: &str, key: &str) -> Result { - log::debug!( - "[memory:driver:embedded] forget namespace={namespace} key_len={}", - key.len() - ); - self.memory() - .await? - .forget(namespace, key) - .await - .map_err(engine_error) - } - - async fn list( - &self, - namespace: Option<&str>, - category: Option<&MemoryCategory>, - session_id: Option<&str>, - ) -> Result, MemoryError> { - let memory = self.memory().await?; - - if let Some(namespace) = namespace { - return memory - .list(Some(namespace), category, session_id) - .await - .map_err(engine_error); - } - - // All-`None` must mean "everything this driver holds" (contract), which - // the engine's namespace normalisation would otherwise narrow to the - // global namespace alone. - let summaries = memory.namespace_summaries().await.map_err(engine_error)?; - log::debug!( - "[memory:driver:embedded] list spanning {} namespace(s)", - summaries.len() - ); - let mut entries = Vec::new(); - for summary in summaries { - let mut page = memory - .list(Some(&summary.namespace), category, session_id) - .await - .map_err(engine_error)?; - entries.append(&mut page); - } - Ok(entries) - } - - async fn namespaces(&self) -> Result, MemoryError> { - // The contract's `namespaces` is the engine's `namespace_summaries`; - // the return type is identical, only the name differs. - self.memory() - .await? - .namespace_summaries() - .await - .map_err(engine_error) - } -} - -#[cfg(test)] -#[path = "core_family_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/core_family_tests.rs b/src/openhuman/memory/driver/embedded/core_family_tests.rs deleted file mode 100644 index 2c1e2feffb..0000000000 --- a/src/openhuman/memory/driver/embedded/core_family_tests.rs +++ /dev/null @@ -1,255 +0,0 @@ -//! [`MemoryCore`] tests. -//! -//! Two carry weight beyond a round-trip: -//! -//! - `store_preserves_external_sync_taint_through_get` is the security test. It -//! asserts the *value*, not merely that a taint exists, so it fails the -//! moment anyone routes the contract's `store` onto `Memory::store` (which -//! hard-codes `Internal`). Its `Internal` twin exists so it cannot pass by a -//! constant. -//! - `list_with_no_namespace_spans_every_namespace` pins the divergence between -//! the contract ("all `None` lists everything") and the engine (`None` -//! normalises to the global namespace). A naive delegation fails it. - -use super::super::test_support::fresh_driver; -use super::*; - -use tinycortex_api::provider::MemoryProvider; - -#[tokio::test] -async fn store_get_round_trips_through_the_contract() { - let (_tmp, provider) = fresh_driver(); - - provider - .store( - "ns_a", - "k1", - "value in a", - MemoryCategory::Core, - Some("sess-1"), - MemoryTaint::Internal, - ) - .await - .expect("store"); - - let got = provider - .get("ns_a", "k1") - .await - .expect("get") - .expect("entry exists"); - assert_eq!(got.key, "k1"); - assert_eq!(got.content, "value in a"); - assert_eq!(got.category, MemoryCategory::Core); -} - -#[tokio::test] -async fn get_returns_none_for_an_absent_key() { - let (_tmp, provider) = fresh_driver(); - assert!(provider.get("ns_a", "nope").await.expect("get").is_none()); -} - -#[tokio::test] -async fn forget_removes_the_entry_and_is_idempotent() { - let (_tmp, provider) = fresh_driver(); - provider - .store( - "ns_a", - "k1", - "value", - MemoryCategory::Core, - None, - MemoryTaint::Internal, - ) - .await - .expect("store"); - - assert!(provider.forget("ns_a", "k1").await.expect("first forget")); - assert!(provider.get("ns_a", "k1").await.expect("get").is_none()); - assert!( - !provider.forget("ns_a", "k1").await.expect("second forget"), - "forgetting an absent key is Ok(false), never an error" - ); -} - -#[tokio::test] -async fn list_scoped_to_namespace_applies_category_and_session_filters() { - let (_tmp, provider) = fresh_driver(); - provider - .store( - "ns_a", - "core-1", - "c", - MemoryCategory::Core, - Some("sess-1"), - MemoryTaint::Internal, - ) - .await - .expect("store core"); - provider - .store( - "ns_a", - "daily-1", - "d", - MemoryCategory::Daily, - Some("sess-2"), - MemoryTaint::Internal, - ) - .await - .expect("store daily"); - - let all = provider.list(Some("ns_a"), None, None).await.expect("list"); - assert_eq!(all.len(), 2); - - let core_only = provider - .list(Some("ns_a"), Some(&MemoryCategory::Core), None) - .await - .expect("list by category"); - assert_eq!(core_only.len(), 1); - assert_eq!(core_only[0].key, "core-1"); - - let session_only = provider - .list(Some("ns_a"), None, Some("sess-2")) - .await - .expect("list by session"); - assert_eq!(session_only.len(), 1); - assert_eq!(session_only[0].key, "daily-1"); -} - -#[tokio::test] -async fn list_with_no_namespace_spans_every_namespace() { - let (_tmp, provider) = fresh_driver(); - provider - .store( - "ns_a", - "a1", - "in a", - MemoryCategory::Core, - None, - MemoryTaint::Internal, - ) - .await - .expect("store a"); - provider - .store( - "ns_b", - "b1", - "in b", - MemoryCategory::Core, - None, - MemoryTaint::Internal, - ) - .await - .expect("store b"); - - let everything = provider.list(None, None, None).await.expect("list all"); - let keys: Vec<&str> = everything.iter().map(|e| e.key.as_str()).collect(); - assert!(keys.contains(&"a1"), "missing ns_a entry: {keys:?}"); - assert!(keys.contains(&"b1"), "missing ns_b entry: {keys:?}"); -} - -#[tokio::test] -async fn namespaces_reports_per_namespace_counts() { - let (_tmp, provider) = fresh_driver(); - for key in ["a1", "a2"] { - provider - .store( - "ns_a", - key, - "x", - MemoryCategory::Core, - None, - MemoryTaint::Internal, - ) - .await - .expect("store"); - } - provider - .store( - "ns_b", - "b1", - "x", - MemoryCategory::Core, - None, - MemoryTaint::Internal, - ) - .await - .expect("store"); - - let summaries = provider.namespaces().await.expect("namespaces"); - let a = summaries - .iter() - .find(|s| s.namespace == "ns_a") - .expect("ns_a summary"); - let b = summaries - .iter() - .find(|s| s.namespace == "ns_b") - .expect("ns_b summary"); - assert_eq!(a.count, 2); - assert_eq!(b.count, 1); -} - -/// SECURITY: the contract stamps provenance before the call; the driver must -/// persist exactly what it was handed. Routing onto `Memory::store` would -/// launder this to `Internal`. -#[tokio::test] -async fn store_preserves_external_sync_taint_through_get() { - let (_tmp, provider) = fresh_driver(); - provider - .store( - "ns_a", - "synced", - "from an external source", - MemoryCategory::Core, - None, - MemoryTaint::ExternalSync, - ) - .await - .expect("store"); - - let got = provider - .get("ns_a", "synced") - .await - .expect("get") - .expect("entry exists"); - assert_eq!(got.taint, MemoryTaint::ExternalSync); -} - -/// The negative half of the taint pair — without it the assertion above could -/// pass against a driver that hard-coded `ExternalSync`. -#[tokio::test] -async fn store_preserves_internal_taint_through_get() { - let (_tmp, provider) = fresh_driver(); - provider - .store( - "ns_a", - "typed", - "written by the user", - MemoryCategory::Core, - None, - MemoryTaint::Internal, - ) - .await - .expect("store"); - - let got = provider - .get("ns_a", "typed") - .await - .expect("get") - .expect("entry exists"); - assert_eq!(got.taint, MemoryTaint::Internal); -} - -#[tokio::test] -async fn core_calls_resolve_the_client_lazily_and_only_once() { - let (_tmp, provider) = fresh_driver(); - assert!( - provider.workspace_dir().parent().is_some(), - "sanity: workspace is nested under the temp dir" - ); - // Before any call the workspace does not exist; the first contract call - // creates it. - assert!(!provider.workspace_dir().exists()); - provider.namespaces().await.expect("namespaces"); - assert!(provider.workspace_dir().exists()); - assert_eq!(provider.driver_id(), "tinycortex"); -} diff --git a/src/openhuman/memory/driver/embedded/diff.rs b/src/openhuman/memory/driver/embedded/diff.rs deleted file mode 100644 index 5962b8e99b..0000000000 --- a/src/openhuman/memory/driver/embedded/diff.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! [`MemoryDiff`] for the embedded driver — snapshot capture and change -//! computation over synced sources. -//! -//! Every method delegates to [`memory::diff::ops`](crate::openhuman::memory::diff::ops), -//! never to `tinycortex::memory::diff::DiffEngine` or `Ledger` directly. That -//! matters for more than tidiness: `ops::take_snapshot` publishes -//! [`DomainEvent::MemoryDiffSnapshotTaken`](crate::core::event_bus::DomainEvent), -//! and reaching past it would make a snapshot captured through the contract -//! invisible to every subscriber that watches for one. -//! -//! ## Three contract methods, ten host functions — the other seven stay host-side -//! -//! [`MemoryDiff`] is exactly `capture_snapshot` / `snapshots` / `diff`. The host -//! additionally has `diff_since_last`, `diff_since_read`, `mark_read`, -//! `create_checkpoint`, `diff_since_checkpoint`, `cleanup` and -//! `auto_snapshot_after_sync`. Those are **not omissions**: read markers and -//! named checkpoints are product surface with no contract representation, and -//! `auto_snapshot_after_sync` is a hook on the host's sync scheduling. They keep -//! their RPC/tool entry points and are not reachable through the provider. -//! -//! ## Asymmetric `NotFound`, on purpose -//! -//! `capture_snapshot` on an unknown source is [`MemoryError::NotFound`]: there -//! is nothing to snapshot, and the contract names that case. `snapshots` on an -//! unknown source is an **empty vector**, which the contract also names — the -//! git ledger has no source registry to consult, so "no snapshots" and "no such -//! source" are the same observation there. Do not "fix" the asymmetry by adding -//! a registry lookup to `snapshots`; it would change a documented outcome. -//! -//! ## The source registry is read through the driver's own config -//! -//! [`registry::get_source_in`] rather than `registry::get_source`: the latter -//! resolves the config path from the process environment, which for a driver -//! bound to workspace B would consult workspace A's source list. -//! -//! ## `diff` checks the source it was told about -//! -//! `ops::compute_diff` takes only snapshot ids — they are globally unique commit -//! SHAs, so it ignores `source_id` entirely and the engine's own cross-source -//! guard only catches `from`/`to` disagreeing with *each other*. A caller can -//! therefore diff source A's two snapshots while naming source B and get a -//! report that looks right. The returned `DiffResult` carries the real -//! `source_id`, so this file compares and rejects the mismatch. - -use async_trait::async_trait; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::{ChangeKind, DiffReport, SnapshotRef, SourceChange}; -use tinycortex_api::provider::MemoryDiff; - -use crate::openhuman::memory::diff::ops; -use crate::openhuman::memory::sources::registry; -use tinycortex::memory::diff::types::{ - ChangeKind as EngineChangeKind, DiffResult, ItemChange, Snapshot, SnapshotTrigger, -}; - -use super::{host_error, EmbeddedMemoryProvider}; - -/// Engine snapshot → contract identity. -/// -/// `source_kind` and `trigger` have no home in [`SnapshotRef`]; the contract -/// exposes identity and counts only. -fn to_snapshot_ref(snapshot: Snapshot) -> SnapshotRef { - SnapshotRef { - id: snapshot.id, - source_id: snapshot.source_id, - label: snapshot.label, - item_count: snapshot.item_count, - taken_at_ms: snapshot.taken_at_ms, - } -} - -/// Two enums, identical wire strings, no shared type — so this is a `match`, -/// not a serde round-trip. The contract's own doc records the equivalence. -fn to_change_kind(kind: EngineChangeKind) -> ChangeKind { - match kind { - EngineChangeKind::Added => ChangeKind::Added, - EngineChangeKind::Removed => ChangeKind::Removed, - EngineChangeKind::Modified => ChangeKind::Modified, - } -} - -/// Engine item change → contract change. `text_diff` is dropped because -/// [`SourceChange`] has no field for it — which is also why this family always -/// asks the engine for `include_text_diff: false` rather than computing a diff -/// nobody can read. -fn to_source_change(change: ItemChange) -> SourceChange { - SourceChange { - item_id: change.item_id, - title: change.title, - kind: to_change_kind(change.kind), - old_content_hash: change.old_content_hash, - new_content_hash: change.new_content_hash, - } -} - -/// Engine diff → contract report. The engine's nested `summary` flattens into -/// the report's four counters; `source_kind` / `source_label` are dropped. -fn to_diff_report(result: DiffResult) -> DiffReport { - DiffReport { - source_id: result.source_id, - from_snapshot_id: result.from_snapshot_id, - to_snapshot_id: result.to_snapshot_id, - added: result.summary.added, - removed: result.summary.removed, - modified: result.summary.modified, - unchanged: result.summary.unchanged, - changes: result.changes.into_iter().map(to_source_change).collect(), - } -} - -#[async_trait] -impl MemoryDiff for EmbeddedMemoryProvider { - async fn capture_snapshot(&self, source_id: &str) -> Result { - log::debug!("[memory:driver:embedded] capture_snapshot source_id={source_id}"); - let config = self.config().await?; - - let Some(source) = registry::get_source_in(config, source_id) - .map_err(|error| host_error("capture_snapshot", error))? - else { - return Err(MemoryError::NotFound(source_id.to_string())); - }; - - // `Manual` and not `Auto`: `Auto` is the trigger the host stamps from - // `auto_snapshot_after_sync`, and it is rendered in the ledger trailer - // and the domain event. A snapshot asked for through the contract was - // asked for explicitly. - ops::take_snapshot(&source, config, SnapshotTrigger::Manual) - .await - .map(to_snapshot_ref) - .map_err(|error| host_error("capture_snapshot", error)) - } - - async fn snapshots( - &self, - source_id: &str, - limit: usize, - ) -> Result, MemoryError> { - // The ledger's limit is a `u32`; saturate rather than wrap. - let limit = u32::try_from(limit).unwrap_or(u32::MAX); - log::debug!("[memory:driver:embedded] snapshots source_id={source_id} limit={limit}"); - - let config = self.config().await?; - ops::list_snapshots(config, Some(source_id), limit) - .await - .map(|snapshots| snapshots.into_iter().map(to_snapshot_ref).collect()) - .map_err(|error| host_error("snapshots", error)) - } - - async fn diff( - &self, - source_id: &str, - from: Option<&str>, - to: &str, - ) -> Result { - log::debug!("[memory:driver:embedded] diff source_id={source_id} from={from:?} to={to}"); - let config = self.config().await?; - - // `include_text_diff: false` — see `to_source_change`. - let result = ops::compute_diff(config, from, to, false) - .await - // The host flattens the engine's error to a `String`, so an unknown - // snapshot id cannot be distinguished from a corrupt ledger here. - // The contract asks for `NotFound` in the first case; getting there - // needs `diff::ops` to stop flattening, which is a host change - // beyond this step. Matching on the message text instead would - // silently reclassify the moment libgit2's wording changes. - .map_err(|error| host_error("diff", error))?; - - if result.source_id != source_id { - return Err(MemoryError::Invalid(format!( - "snapshot '{to}' belongs to source '{}', not '{source_id}'", - result.source_id - ))); - } - - Ok(to_diff_report(result)) - } -} - -#[cfg(test)] -#[path = "diff_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/diff_tests.rs b/src/openhuman/memory/driver/embedded/diff_tests.rs deleted file mode 100644 index 1ded041282..0000000000 --- a/src/openhuman/memory/driver/embedded/diff_tests.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! [`MemoryDiff`] tests. -//! -//! Snapshots are seeded straight through the ledger (the same trick -//! `diff::ops`' own tests use) rather than through `capture_snapshot`, because -//! capturing needs a populated chunk store and the mapping under test is the -//! ledger→contract one. -//! -//! Two tests carry weight beyond shape: -//! -//! - `diff_rejects_a_snapshot_belonging_to_another_source` covers the hole -//! `ops::compute_diff` leaves open — it never looks at `source_id`, so without -//! the driver's check a caller can name source B while diffing source A's -//! snapshots and get a plausible report. -//! - `snapshots_on_an_unknown_source_is_empty_not_an_error` pins the deliberate -//! asymmetry with `capture_snapshot`'s `NotFound`. - -use super::super::test_support::fresh_driver; -use super::*; - -use tinycortex::memory::diff::{Ledger, SnapshotMeta}; - -use crate::openhuman::memory::driver::embedded::EmbeddedMemoryProvider; - -/// Commit a snapshot directly into the ledger for `source_id`. -fn seed( - provider: &EmbeddedMemoryProvider, - source_id: &str, - at_ms: i64, - items: &[(&str, &str)], -) -> Snapshot { - let ledger = Ledger::open(provider.workspace_dir()).expect("ledger opens"); - let items: Vec<(String, String)> = items - .iter() - .map(|(k, v)| ((*k).to_string(), (*v).to_string())) - .collect(); - ledger - .commit_snapshot( - &SnapshotMeta { - source_id: source_id.to_string(), - source_kind: "folder".to_string(), - label: "Docs".to_string(), - trigger: SnapshotTrigger::Auto, - }, - &items, - at_ms, - ) - .expect("commit snapshot") -} - -#[tokio::test] -async fn capture_snapshot_on_an_unknown_source_is_not_found() { - let (_tmp, provider) = fresh_driver(); - let error = provider - .capture_snapshot("no-such-source") - .await - .expect_err("unknown source must not silently succeed"); - assert!( - matches!(error, MemoryError::NotFound(ref id) if id == "no-such-source"), - "got: {error:?}" - ); -} - -#[tokio::test] -async fn snapshots_on_an_unknown_source_is_empty_not_an_error() { - let (_tmp, provider) = fresh_driver(); - let snapshots = provider - .snapshots("no-such-source", 10) - .await - .expect("an unknown source yields an empty list, per the contract"); - assert!(snapshots.is_empty()); -} - -#[tokio::test] -async fn snapshots_maps_ledger_entries_onto_the_contract_shape() { - let (_tmp, provider) = fresh_driver(); - seed(&provider, "src_a", 1_000, &[("a", "alpha")]); - seed(&provider, "src_a", 2_000, &[("a", "alpha"), ("b", "beta")]); - // A second source must not leak into the first source's listing. - seed(&provider, "src_b", 3_000, &[("z", "zeta")]); - - let snapshots = provider.snapshots("src_a", 10).await.expect("snapshots"); - - assert_eq!(snapshots.len(), 2); - assert!(snapshots.iter().all(|s| s.source_id == "src_a")); - // Newest first, per the trait doc. - assert_eq!(snapshots[0].taken_at_ms, 2_000); - assert_eq!(snapshots[0].item_count, 2); - assert_eq!(snapshots[0].label, "Docs"); - assert!(!snapshots[0].id.is_empty()); -} - -#[tokio::test] -async fn snapshots_honours_the_limit() { - let (_tmp, provider) = fresh_driver(); - seed(&provider, "src_a", 1_000, &[("a", "alpha")]); - seed(&provider, "src_a", 2_000, &[("a", "beta")]); - - let snapshots = provider.snapshots("src_a", 1).await.expect("snapshots"); - assert_eq!(snapshots.len(), 1); -} - -#[tokio::test] -async fn diff_counts_and_per_item_kinds_match_the_ledger() { - let (_tmp, provider) = fresh_driver(); - let from = seed( - &provider, - "src_a", - 1_000, - &[("a", "alpha"), ("b", "beta"), ("c", "gamma")], - ); - let to = seed( - &provider, - "src_a", - 2_000, - &[("a", "alpha"), ("b", "beta v2"), ("d", "delta")], - ); - - let report = provider - .diff("src_a", Some(&from.id), &to.id) - .await - .expect("diff"); - - assert_eq!(report.source_id, "src_a"); - assert_eq!(report.from_snapshot_id.as_deref(), Some(from.id.as_str())); - assert_eq!(report.to_snapshot_id, to.id); - assert_eq!(report.added, 1); - assert_eq!(report.removed, 1); - assert_eq!(report.modified, 1); - assert_eq!(report.unchanged, 1); - - let kind_of = |id: &str| { - report - .changes - .iter() - .find(|c| c.item_id == id) - .map(|c| c.kind) - }; - assert_eq!(kind_of("d"), Some(ChangeKind::Added)); - assert_eq!(kind_of("c"), Some(ChangeKind::Removed)); - assert_eq!(kind_of("b"), Some(ChangeKind::Modified)); - assert_eq!(kind_of("a"), None, "unchanged items are not listed"); -} - -#[tokio::test] -async fn diff_with_no_baseline_reports_everything_added() { - let (_tmp, provider) = fresh_driver(); - let to = seed(&provider, "src_a", 1_000, &[("a", "alpha")]); - - let report = provider.diff("src_a", None, &to.id).await.expect("diff"); - assert_eq!(report.added, 1); - assert_eq!(report.from_snapshot_id, None); -} - -#[tokio::test] -async fn diff_rejects_a_snapshot_belonging_to_another_source() { - let (_tmp, provider) = fresh_driver(); - let from = seed(&provider, "src_a", 1_000, &[("a", "alpha")]); - let to = seed(&provider, "src_a", 2_000, &[("a", "beta")]); - - // Both snapshots really are src_a's, so the engine's own cross-source guard - // is satisfied — only the driver's check catches the wrong source name. - let error = provider - .diff("src_b", Some(&from.id), &to.id) - .await - .expect_err("naming the wrong source must not produce a plausible report"); - assert!( - matches!(error, MemoryError::Invalid(ref message) if message.contains("src_a")), - "got: {error:?}" - ); -} diff --git a/src/openhuman/memory/driver/embedded/documents.rs b/src/openhuman/memory/driver/embedded/documents.rs deleted file mode 100644 index 6dde2787f7..0000000000 --- a/src/openhuman/memory/driver/embedded/documents.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! [`MemoryDocuments`] for the embedded driver — the namespace-document tier. -//! -//! ## The contract types *are* the host types -//! -//! `NamespaceDocumentInput`, `StoredMemoryDocument` and -//! `NamespaceRetrievalContext` in -//! [`crate::openhuman::memory::store::types`] are `pub use`d from -//! `tinycortex::memory`, which in turn re-exports `tinycortex_api::types`. -//! Same crate, same types — so there is no conversion in this file, only -//! signature shape (`usize` → `u32`, `Result<_, String>` → -//! [`MemoryError`]). -//! -//! ## `put_document` keeps the full pipeline -//! -//! It delegates to `MemoryClient::put_doc`, which persists and then enqueues a -//! background graph-extraction job — not `put_doc_light`, which skips vector -//! and graph indexing. The contract says nothing about extraction, so the -//! driver inherits the host's normal write behaviour rather than quietly -//! choosing the cheaper one. -//! -//! ## `get_document` had no host entry point -//! -//! There was no read-one-by-key path anywhere: `MemoryClient` has -//! `put_doc` / `list_documents` / `delete_document`, and `list_documents`' -//! SELECT carries no `content` column. `UnifiedMemory::get_document_by_key` -//! (added alongside this family) is `load_documents_for_scope`'s SELECT with a -//! `key` predicate — a filter on an existing query, not retrieval logic. It -//! canonicalizes the key through -//! [`canonical_document_key`](crate::openhuman::memory::store::safety::canonical_document_key), -//! the same transform the write path applies, so a PII-shaped key written by -//! `put_document` reads back rather than silently missing (#5164). - -use async_trait::async_trait; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::MemoryDocuments; -use tinycortex_api::types::{ - NamespaceDocumentInput, NamespaceRetrievalContext, StoredMemoryDocument, -}; - -use super::{host_error, EmbeddedMemoryProvider}; - -#[async_trait] -impl MemoryDocuments for EmbeddedMemoryProvider { - async fn put_document(&self, input: NamespaceDocumentInput) -> Result { - log::debug!( - "[memory:driver:embedded] put_document namespace={} key_chars={} content_chars={}", - input.namespace, - input.key.chars().count(), - input.content.chars().count() - ); - self.client() - .await? - .put_doc(input) - .await - .map_err(|error| host_error("put_document", error)) - } - - async fn get_document( - &self, - namespace: &str, - key: &str, - ) -> Result, MemoryError> { - log::debug!( - "[memory:driver:embedded] get_document namespace={namespace} key_chars={}", - key.chars().count() - ); - self.client() - .await? - .get_document(namespace, key) - .await - .map_err(|error| host_error("get_document", error)) - } - - async fn query_documents( - &self, - namespace: &str, - query: &str, - limit: usize, - ) -> Result { - // The host's chunk budget is a `u32`; saturate rather than wrap. - let max_chunks = u32::try_from(limit).unwrap_or(u32::MAX); - log::debug!( - "[memory:driver:embedded] query_documents namespace={namespace} query_len={} \ - max_chunks={max_chunks}", - query.len() - ); - self.client() - .await? - .query_namespace_context_data(namespace, query, max_chunks) - .await - .map_err(|error| host_error("query_documents", error)) - } -} - -#[cfg(test)] -#[path = "documents_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/documents_tests.rs b/src/openhuman/memory/driver/embedded/documents_tests.rs deleted file mode 100644 index 4df1c1baa2..0000000000 --- a/src/openhuman/memory/driver/embedded/documents_tests.rs +++ /dev/null @@ -1,194 +0,0 @@ -//! [`MemoryDocuments`] tests. -//! -//! Two carry weight beyond a round-trip: -//! -//! - `get_document_finds_a_key_the_write_path_canonicalized` is the #5164 -//! regression. The write path rewrites a PII-shaped key before storing it, so -//! a read that addresses the raw key misses, the caller treats the row as -//! absent, and writes it again. It passes only because -//! `get_document_by_key` canonicalizes through the same helper. -//! - `put_document_through_the_contract_is_visible_to_list_documents` is the -//! same-store proof: the contract write lands where the existing RPC read -//! path looks, not in a parallel store. - -use super::super::test_support::fresh_driver; -use super::*; - -use serde_json::json; - -fn doc(namespace: &str, key: &str, title: &str, content: &str) -> NamespaceDocumentInput { - NamespaceDocumentInput { - namespace: namespace.to_string(), - key: key.to_string(), - title: title.to_string(), - content: content.to_string(), - source_type: "chat".to_string(), - priority: "normal".to_string(), - tags: vec!["t1".to_string()], - metadata: json!({"origin": "test"}), - category: "core".to_string(), - session_id: None, - document_id: None, - taint: Default::default(), - } -} - -#[tokio::test] -async fn put_document_then_get_document_returns_the_same_body() { - let (_tmp, provider) = fresh_driver(); - - let id = provider - .put_document(doc("docs_ns", "readme", "Readme", "the whole body")) - .await - .expect("put_document"); - assert!(!id.is_empty(), "put_document must return a document id"); - - let got = provider - .get_document("docs_ns", "readme") - .await - .expect("get_document") - .expect("document exists"); - - assert_eq!(got.key, "readme"); - assert_eq!(got.title, "Readme"); - // The body is the point: `list_documents` cannot back this method because - // its SELECT has no `content` column. - assert_eq!(got.content, "the whole body"); - assert_eq!(got.tags, vec!["t1".to_string()]); - assert_eq!(got.metadata, json!({"origin": "test"})); -} - -#[tokio::test] -async fn put_document_upserts_on_the_same_key() { - let (_tmp, provider) = fresh_driver(); - - provider - .put_document(doc("docs_ns", "readme", "First", "first body")) - .await - .expect("first put"); - provider - .put_document(doc("docs_ns", "readme", "Second", "second body")) - .await - .expect("second put"); - - let got = provider - .get_document("docs_ns", "readme") - .await - .expect("get_document") - .expect("document exists"); - assert_eq!(got.content, "second body"); -} - -#[tokio::test] -async fn get_document_returns_none_for_unknown_key() { - let (_tmp, provider) = fresh_driver(); - assert!(provider - .get_document("docs_ns", "nope") - .await - .expect("get_document") - .is_none()); -} - -#[tokio::test] -async fn get_document_finds_a_key_the_write_path_canonicalized() { - use crate::openhuman::memory::store::safety::canonical_document_key; - - let (_tmp, provider) = fresh_driver(); - // A strict-gated PII shape — `canonical_identifier` deliberately leaves - // scanner-built identifiers (JIDs, E.164 chat ids, timestamps) alone. - let raw_key = "ssn-123-45-6789"; - // Guard the premise: if this key stops being rewritten the test still - // passes but stops testing anything, so assert the rewrite happens. - assert_ne!( - canonical_document_key(raw_key), - raw_key, - "this key must be PII-shaped for the regression to mean anything" - ); - - provider - .put_document(doc("docs_ns", raw_key, "Contact", "a phone number")) - .await - .expect("put_document"); - - let got = provider - .get_document("docs_ns", raw_key) - .await - .expect("get_document"); - assert!( - got.is_some(), - "a canonicalized key must stay addressable by its raw form (#5164)" - ); -} - -#[tokio::test] -async fn query_documents_returns_hits_and_rendered_context() { - let (_tmp, provider) = fresh_driver(); - - provider - .put_document(doc( - "docs_ns", - "kettle", - "Kettle", - "the kettle boils at one hundred degrees", - )) - .await - .expect("put_document"); - - let context = provider - .query_documents("docs_ns", "kettle", 5) - .await - .expect("query_documents"); - - assert_eq!(context.namespace, "docs_ns"); - assert_eq!(context.query.as_deref(), Some("kettle")); - assert!( - !context.hits.is_empty(), - "the stored document must be retrievable" - ); - assert!( - !context.context_text.is_empty(), - "the driver must return the host's rendered context, not re-assemble it" - ); -} - -#[tokio::test] -async fn query_documents_on_an_empty_namespace_is_not_an_error() { - let (_tmp, provider) = fresh_driver(); - let context = provider - .query_documents("empty_ns", "anything", 5) - .await - .expect("query_documents"); - assert!(context.hits.is_empty()); -} - -#[tokio::test] -async fn put_document_through_the_contract_is_visible_to_list_documents() { - let (_tmp, provider) = fresh_driver(); - - provider - .put_document(doc("docs_ns", "shared", "Shared", "body")) - .await - .expect("put_document"); - - // The existing RPC read path, reached through the same client. - let listed = provider - .client() - .await - .expect("client") - .list_documents(Some("docs_ns")) - .await - .expect("list_documents"); - let keys: Vec = listed - .get("documents") - .and_then(serde_json::Value::as_array) - .expect("documents array") - .iter() - .filter_map(|d| d.get("key").and_then(serde_json::Value::as_str)) - .map(str::to_string) - .collect(); - - assert!( - keys.contains(&"shared".to_string()), - "contract write must land in the store the RPC path reads: {keys:?}" - ); -} diff --git a/src/openhuman/memory/driver/embedded/entities.rs b/src/openhuman/memory/driver/embedded/entities.rs deleted file mode 100644 index bc54565557..0000000000 --- a/src/openhuman/memory/driver/embedded/entities.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! [`MemoryEntities`] for the embedded driver — the entity index and its -//! hotness counters. -//! -//! ## Two rankers behind one method -//! -//! The contract's [`MemoryEntities::entities`] ranks "by hotness when `query` -//! is `None` and by match quality otherwise". Those are two different host -//! surfaces, not one with a flag: -//! -//! - `query = Some` → `tree::retrieval::search_entities`, the engine's ranked -//! surface-form search, returning `EntityMatch`. -//! - `query = None` → `read_rpc::top_entities_rpc`, a `GROUP BY` over -//! `mem_tree_entity_index` ordered by mention count then recency, returning -//! `read_rpc::types::EntityRef`. -//! -//! ## Where `hotness` comes from -//! -//! Neither ranker computes it — both are pure SQL over the index. The hotness -//! signal lives in a separate table (`mem_tree_entity_hotness`, read through -//! `store::trees::hotness::get`) and is turned into a scalar by -//! `TreePolicy::topic_hotness`, the host's existing formula. This file calls -//! that formula; it does not define one. An entity with no hotness row scores -//! `0.0`, which is what `topic_hotness` returns for zero signal anyway. -//! -//! That costs one extra read per returned row. It is bounded by the `limit` -//! already applied by the ranker, and there is no batch getter below this line -//! to use instead. -//! -//! ## `entity_edges` is a projection of the co-occurrence table — read this -//! -//! There is **no host function that returns a `GraphRelationRecord` for an -//! entity-index id.** Two things look like one and are not: -//! -//! - `memory::store::namespace_store::graph::graph_relations_namespace` is the -//! *namespace-document* graph (subject/predicate/object extracted from -//! documents). Its subjects are document entity strings, not -//! `mem_tree_entity_index` canonical ids, so joining the two would silently -//! mix id spaces. That surface is already exposed properly, as -//! [`MemoryGraph::relations`](tinycortex_api::provider::MemoryGraph::relations). -//! - `memory::tree::graph::store::neighbors` is the *entity-index* -//! co-occurrence table, keyed by exactly the right id — but it is undirected -//! and carries only a weight. -//! -//! `neighbors` is the honest backing, so this method projects it into the -//! contract shape with a **single fixed predicate**, `"co_occurs_with"`. -//! `evidence_count` is the real co-occurrence count; `attrs` is `null`, -//! `updated_at` is `0.0`, and `document_ids` / `chunk_ids` are empty because -//! the table stores none of them. A reader who sees `GraphRelationRecord` here -//! must not assume the richer graph tier — that is what this paragraph is for. - -use async_trait::async_trait; -use chrono::Utc; -use serde_json::Value; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::{EntityHit, EntityRef}; -use tinycortex_api::provider::MemoryEntities; -use tinycortex_api::types::GraphRelationRecord; - -use crate::openhuman::config::Config; -use crate::openhuman::memory::read_rpc; -use crate::openhuman::memory::store::trees::hotness; -use crate::openhuman::memory::tree::graph::store as graph_store; -use crate::openhuman::memory::tree::retrieval::search_entities; -use crate::openhuman::memory::tree_policy::TreePolicy; - -use super::{host_error, EmbeddedMemoryProvider}; - -/// The predicate materialised for every projected co-occurrence edge. -/// -/// A constant rather than an inline literal so a caller can match on it and a -/// test can assert it without duplicating the string. -pub(super) const CO_OCCURRENCE_PREDICATE: &str = "co_occurs_with"; - -/// The host's hotness scalar for one entity, or `0.0` when it has no counters. -/// -/// Blocking (SQLite) — call from the blocking pool. -fn hotness_for(config: &Config, entity_id: &str) -> f64 { - match hotness::get(config, entity_id) { - Ok(Some(counters)) => TreePolicy::topic().topic_hotness( - entity_id, - &counters.stats(), - Utc::now().timestamp_millis(), - ) as f64, - Ok(None) => 0.0, - Err(error) => { - // A missing hotness row is not an error, and neither is a failed - // read: ranking degrades, the entity list does not disappear. - log::warn!("[memory:driver:embedded] hotness read failed: {error:#}"); - 0.0 - } - } -} - -/// Attaches hotness to a batch of `(id, kind, name, mentions)` tuples. -async fn with_hotness( - config: &Config, - rows: Vec<(String, String, String, u32)>, -) -> Result, MemoryError> { - let config = config.clone(); - tokio::task::spawn_blocking(move || { - rows.into_iter() - .map(|(id, kind, name, mentions)| { - let hotness = hotness_for(&config, &id); - EntityHit { - entity: EntityRef { id, kind, name }, - hotness, - mentions, - } - }) - .collect() - }) - .await - .map_err(|error| host_error("entities_hotness", format!("join error: {error}"))) -} - -#[async_trait] -impl MemoryEntities for EmbeddedMemoryProvider { - async fn entities( - &self, - namespace: &str, - query: Option<&str>, - limit: usize, - ) -> Result, MemoryError> { - log::debug!( - "[memory:driver:embedded] entities namespace={namespace} has_query={} limit={limit}", - query.is_some() - ); - // `mem_tree_entity_index` has no namespace column — the index is - // process-wide within a workspace. `namespace` is accepted for - // contract shape and deliberately not used as a filter; inventing one - // would be a new predicate, not a delegation. - let _ = namespace; - - let config = self.config().await?; - let rows = match query { - Some(query) => search_entities(config, query, None, limit) - .await - .map_err(|error| host_error("entities_search", format!("{error:#}")))? - .into_iter() - .map(|hit| { - ( - hit.canonical_id, - hit.kind.as_str().to_string(), - hit.surface, - u32::try_from(hit.mention_count).unwrap_or(u32::MAX), - ) - }) - .collect::>(), - None => { - read_rpc::top_entities_rpc(config, None, u32::try_from(limit).unwrap_or(u32::MAX)) - .await - .map_err(|error| host_error("entities_top", error))? - .value - .into_iter() - .map(|entity| (entity.entity_id, entity.kind, entity.surface, entity.count)) - .collect::>() - } - }; - - with_hotness(config, rows).await - } - - async fn entity_edges( - &self, - namespace: &str, - entity_id: &str, - limit: usize, - ) -> Result, MemoryError> { - log::debug!("[memory:driver:embedded] entity_edges namespace={namespace} limit={limit}"); - // Same reason as `entities`: the co-occurrence table is not - // namespace-keyed. - let _ = namespace; - - let config = self.config().await?.clone(); - let subject = entity_id.to_string(); - let neighbours = tokio::task::spawn_blocking(move || { - graph_store::neighbors(&config, &subject).map(|mut rows| { - // `neighbors` has no limit parameter, so the ceiling is applied - // here. Rows already arrive weight-descending from the engine's - // query; truncation therefore keeps the strongest edges. - rows.truncate(limit); - rows - }) - }) - .await - .map_err(|error| host_error("entity_edges", format!("join error: {error}")))? - .map_err(|error| host_error("entity_edges", format!("{error:#}")))?; - - // An unknown entity yields no rows, which the contract says must be an - // empty vector rather than `NotFound`. - Ok(neighbours - .into_iter() - .map(|(object, weight)| GraphRelationRecord { - namespace: None, - subject: entity_id.to_string(), - predicate: CO_OCCURRENCE_PREDICATE.to_string(), - object, - attrs: Value::Null, - updated_at: 0.0, - evidence_count: u32::try_from(weight.max(0)).unwrap_or(u32::MAX), - order_index: None, - document_ids: Vec::new(), - chunk_ids: Vec::new(), - }) - .collect()) - } - - async fn touch_entities( - &self, - namespace: &str, - entity_ids: &[String], - ) -> Result<(), MemoryError> { - log::debug!( - "[memory:driver:embedded] touch_entities namespace={namespace} n={}", - entity_ids.len() - ); - let _ = namespace; - if entity_ids.is_empty() { - return Ok(()); - } - - let config = self.config().await?.clone(); - let entity_ids = entity_ids.to_vec(); - tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - let now_ms = Utc::now().timestamp_millis(); - for entity_id in &entity_ids { - // `get_or_fresh` creates a zeroed row for an id the index has - // never seen, which is how "unknown ids are ignored, not - // rejected" is satisfied without a pre-existence check. - let mut counters = hotness::get_or_fresh(&config, entity_id)?; - counters.mention_count_30d = counters.mention_count_30d.saturating_add(1); - counters.last_seen_ms = Some(now_ms); - counters.last_updated_ms = now_ms; - hotness::upsert(&config, &counters)?; - } - Ok(()) - }) - .await - .map_err(|error| host_error("touch_entities", format!("join error: {error}")))? - .map_err(|error| host_error("touch_entities", format!("{error:#}"))) - } -} - -#[cfg(test)] -#[path = "entities_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/entities_tests.rs b/src/openhuman/memory/driver/embedded/entities_tests.rs deleted file mode 100644 index 48d4ccbd7c..0000000000 --- a/src/openhuman/memory/driver/embedded/entities_tests.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! [`MemoryEntities`] tests for the embedded driver. - -use super::super::test_support::fresh_driver; -use super::CO_OCCURRENCE_PREDICATE; - -use chrono::Utc; -use tinycortex_api::provider::MemoryEntities; - -use crate::openhuman::config::Config; -use crate::openhuman::memory::store::entities::index_entity; -use crate::openhuman::memory::store::trees::hotness; -use crate::openhuman::memory::tree::graph::store as graph_store; -use tinycortex::memory::store::entity_index::{CanonicalEntity, EntityKind}; - -/// Seed one occurrence into the entity index through the host's own writer. -/// -/// Deliberately not raw SQL: `mem_tree_entity_index` is engine-owned and its -/// column set has already moved once, so a hand-written INSERT here would rot -/// against a schema change instead of following it. -fn seed_entity(config: &Config, entity_id: &str, kind: EntityKind, surface: &str, node_id: &str) { - index_entity( - config, - &CanonicalEntity { - canonical_id: entity_id.to_string(), - kind, - surface: surface.to_string(), - span_start: 0, - span_end: u32::try_from(surface.len()).unwrap_or(1), - score: 1.0, - }, - node_id, - "chunk", - Utc::now().timestamp_millis(), - None, - ) - .expect("seed entity index row"); -} - -#[tokio::test] -async fn entities_unknown_workspace_yields_empty() { - let (_tmp, provider) = fresh_driver(); - let hits = provider.entities("work", None, 10).await.expect("entities"); - assert!(hits.is_empty(), "an empty index ranks nothing"); -} - -#[tokio::test] -async fn entities_ranked_by_mentions_when_query_absent() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - seed_entity(&config, "topic:phoenix", EntityKind::Topic, "Phoenix", "n1"); - seed_entity(&config, "topic:phoenix", EntityKind::Topic, "Phoenix", "n2"); - seed_entity(&config, "topic:atlas", EntityKind::Topic, "Atlas", "n3"); - - let hits = provider.entities("work", None, 10).await.expect("entities"); - assert_eq!(hits.len(), 2); - assert_eq!(hits[0].entity.id, "topic:phoenix", "most mentions first"); - assert_eq!(hits[0].mentions, 2); - assert_eq!(hits[0].entity.kind, "topic"); - assert_eq!(hits[0].entity.name, "Phoenix"); -} - -#[tokio::test] -async fn entities_ranked_by_match_when_query_present() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - seed_entity(&config, "topic:phoenix", EntityKind::Topic, "Phoenix", "n1"); - seed_entity(&config, "topic:atlas", EntityKind::Topic, "Atlas", "n2"); - - let hits = provider - .entities("work", Some("Phoenix"), 10) - .await - .expect("entities"); - assert!( - hits.iter().any(|hit| hit.entity.id == "topic:phoenix"), - "the matching entity must be returned, got {:?}", - hits.iter().map(|h| &h.entity.id).collect::>() - ); - assert!( - !hits.iter().any(|hit| hit.entity.id == "topic:atlas"), - "a non-matching entity must not be" - ); -} - -#[tokio::test] -async fn entities_hotness_reflects_the_hotness_table() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - seed_entity(&config, "topic:phoenix", EntityKind::Topic, "Phoenix", "n1"); - - let cold = provider.entities("work", None, 10).await.expect("entities"); - assert_eq!( - cold[0].hotness, 0.0, - "an entity with no hotness row scores zero" - ); - - provider - .touch_entities("work", &["topic:phoenix".to_string()]) - .await - .expect("touch_entities"); - - let warm = provider.entities("work", None, 10).await.expect("entities"); - assert!( - warm[0].hotness > 0.0, - "touching the entity must raise its hotness, got {}", - warm[0].hotness - ); -} - -#[tokio::test] -async fn touch_entities_bumps_hotness_counters() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - - provider - .touch_entities("work", &["topic:phoenix".to_string()]) - .await - .expect("first touch"); - provider - .touch_entities("work", &["topic:phoenix".to_string()]) - .await - .expect("second touch"); - - let counters = hotness::get(&config, "topic:phoenix") - .expect("hotness read") - .expect("row exists after touching"); - assert_eq!(counters.mention_count_30d, 2); - assert!(counters.last_seen_ms.is_some()); -} - -#[tokio::test] -async fn touch_entities_accepts_unknown_ids_rather_than_rejecting_them() { - let (_tmp, provider) = fresh_driver(); - provider - .touch_entities("work", &["entity:never-seen".to_string()]) - .await - .expect("the contract says unknown ids are ignored, not rejected"); - provider - .touch_entities("work", &[]) - .await - .expect("an empty list is a no-op"); -} - -#[tokio::test] -async fn entity_edges_projects_cooccurrence_neighbours() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - graph_store::upsert_edges( - &config, - &[("topic:phoenix".to_string(), "person:alice".to_string())], - Utc::now().timestamp_millis(), - ) - .expect("seed co-occurrence edge"); - - let edges = provider - .entity_edges("work", "topic:phoenix", 10) - .await - .expect("entity_edges"); - - assert_eq!(edges.len(), 1); - assert_eq!(edges[0].subject, "topic:phoenix"); - assert_eq!(edges[0].object, "person:alice"); - assert_eq!( - edges[0].predicate, CO_OCCURRENCE_PREDICATE, - "the projection materialises one fixed predicate" - ); - assert!( - edges[0].evidence_count >= 1, - "the co-occurrence weight is the evidence count" - ); - assert!(edges[0].document_ids.is_empty()); - assert!(edges[0].chunk_ids.is_empty()); -} - -#[tokio::test] -async fn entity_edges_unknown_entity_is_empty_not_not_found() { - let (_tmp, provider) = fresh_driver(); - let edges = provider - .entity_edges("work", "topic:never-seen", 10) - .await - .expect("'no edges' and 'no such entity' are the same answer"); - assert!(edges.is_empty()); -} diff --git a/src/openhuman/memory/driver/embedded/goals.rs b/src/openhuman/memory/driver/embedded/goals.rs deleted file mode 100644 index ca5cfa6a8b..0000000000 --- a/src/openhuman/memory/driver/embedded/goals.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! [`MemoryGoals`] for the embedded driver — the agent's long-term goals -//! document. -//! -//! The smallest family in the contract, and the only one with **no type -//! conversion at all**: `tinycortex::memory::goals::types` is a `pub use` of -//! `tinycortex_api::goals` (see `vendor/tinycortex/src/memory/goals/mod.rs`), -//! so the [`GoalsDoc`] the host store reads and writes *is* the contract's -//! [`GoalsDoc`]. `goals_doc_is_the_contract_type` pins that — if the engine -//! ever forks the type, this file should stop compiling here rather than -//! somewhere confusing. -//! -//! ## Both directions go through the engine store -//! -//! `store::load` / `store::save` are `tinycortex::memory::goals::store`. They -//! own the on-disk location (`/MEMORY_GOALS.md`) and the -//! item/character caps, and going through them keeps this driver from being a -//! second place that knows either. -//! -//! ## `set_goals` takes ownership; `save` needs `&mut` -//! -//! `store::save` trims the document in place to `GOALS_MAX_ITEMS` / -//! `GOALS_FILE_MAX_CHARS`. The contract hands the document over by value and -//! returns `()`, so the trimmed copy is simply dropped — the caller's next -//! [`MemoryGoals::goals`] reads whatever was actually persisted, which is the -//! honest answer. -//! -//! ## Why nothing maps to [`MemoryError::Invalid`] -//! -//! The contract reserves `Invalid` for "a document the driver refuses (e.g. -//! over its own item cap)". The engine *does* have those rejections, and now -//! that the host shim is gone they arrive here as a typed -//! `tinycortex::memory::error::MemoryError`. This file still flattens them with -//! `to_string()` into [`MemoryError::Other`], because the facade collapse is a -//! pure relocation; mapping engine `Invalid`/`NotFound` onto the contract's -//! variants is a behaviour change and is tracked separately. - -use async_trait::async_trait; -use tinycortex_api::error::MemoryError; -use tinycortex_api::goals::GoalsDoc; -use tinycortex_api::provider::MemoryGoals; - -use tinycortex::memory::goals::store; - -use super::{host_error, EmbeddedMemoryProvider}; - -#[async_trait] -impl MemoryGoals for EmbeddedMemoryProvider { - async fn goals(&self) -> Result { - log::debug!( - "[memory:driver:embedded] goals workspace={}", - self.workspace_dir().display() - ); - // A missing `MEMORY_GOALS.md` maps to an empty document inside - // `store::load`, so the contract's "no goals is not NotFound" rule - // holds without anything here. - store::load(self.workspace_dir()).map_err(|error| host_error("goals", error.to_string())) - } - - async fn set_goals(&self, goals: GoalsDoc) -> Result<(), MemoryError> { - let mut doc = goals; - log::debug!( - "[memory:driver:embedded] set_goals workspace={} items={}", - self.workspace_dir().display(), - doc.items.len() - ); - store::save(self.workspace_dir(), &mut doc) - .map_err(|error| host_error("set_goals", error.to_string())) - } -} - -#[cfg(test)] -#[path = "goals_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/goals_tests.rs b/src/openhuman/memory/driver/embedded/goals_tests.rs deleted file mode 100644 index b8b2c89b01..0000000000 --- a/src/openhuman/memory/driver/embedded/goals_tests.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! [`MemoryGoals`] tests. -//! -//! `goals_doc_is_the_contract_type` is the one that outlives the round-trip: the -//! whole family is conversion-free only because the engine re-exports the -//! contract's `GoalsDoc`. If that ever forks, this file should say so. - -use super::super::test_support::fresh_driver; -use super::*; - -use tinycortex_api::goals::GoalItem; - -#[test] -fn goals_doc_is_the_contract_type() { - // Not a tautology: `store::load` is typed on - // `tinycortex::memory::goals::types::GoalsDoc`, and this assignment only - // compiles while that is a re-export of the contract type. - let engine: tinycortex::memory::goals::types::GoalsDoc = Default::default(); - let _contract: GoalsDoc = engine; -} - -#[tokio::test] -async fn goals_on_a_fresh_workspace_is_empty_not_not_found() { - let (_tmp, provider) = fresh_driver(); - let doc = provider.goals().await.expect("goals must not be NotFound"); - assert!(doc.items.is_empty()); -} - -#[tokio::test] -async fn set_goals_then_goals_round_trips() { - let (_tmp, provider) = fresh_driver(); - - let doc = GoalsDoc { - items: vec![ - GoalItem::new("g1", "ship the memory contract"), - GoalItem::new("g2", "keep the build green"), - ], - }; - provider.set_goals(doc.clone()).await.expect("set_goals"); - - let read_back = provider.goals().await.expect("goals"); - assert_eq!(read_back, doc); -} - -#[tokio::test] -async fn set_goals_replaces_wholesale_rather_than_merging() { - let (_tmp, provider) = fresh_driver(); - - provider - .set_goals(GoalsDoc { - items: vec![GoalItem::new("g1", "first")], - }) - .await - .expect("first set_goals"); - provider - .set_goals(GoalsDoc { - items: vec![GoalItem::new("g2", "second")], - }) - .await - .expect("second set_goals"); - - let read_back = provider.goals().await.expect("goals"); - assert_eq!(read_back.items.len(), 1, "whole-document replacement"); - assert_eq!(read_back.items[0].text, "second"); -} - -#[tokio::test] -async fn set_goals_writes_the_host_file_the_rest_of_the_product_reads() { - let (_tmp, provider) = fresh_driver(); - provider - .set_goals(GoalsDoc { - items: vec![GoalItem::new("g1", "visible to the host")], - }) - .await - .expect("set_goals"); - - // Same-store proof: the contract write must land where the existing RPC / - // agent-tool readers look, not in a parallel file. - let path = store::goals_path(provider.workspace_dir()); - let body = std::fs::read_to_string(&path).expect("MEMORY_GOALS.md exists"); - assert!(body.contains("visible to the host"), "got: {body}"); -} diff --git a/src/openhuman/memory/driver/embedded/graph.rs b/src/openhuman/memory/driver/embedded/graph.rs deleted file mode 100644 index 0e8a60ebac..0000000000 --- a/src/openhuman/memory/driver/embedded/graph.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! [`MemoryGraph`] for the embedded driver — the key/value and relation tier. -//! -//! ## Which host method backs which contract method -//! -//! The obvious `MemoryClient` methods are the wrong ones here, and it is worth -//! saying why rather than re-deriving it later: -//! -//! - `kv_get` returns a bare `serde_json::Value`; the contract wants a -//! [`MemoryKvRecord`], which also carries `updated_at`. That timestamp is -//! simply not on the value path — `KvStore::get_*` drops it too. -//! - `kv_list_namespace` returns `Vec`, takes `&str` rather -//! than `Option<&str>` (so it cannot address the global slice), and has no -//! prefix or limit. -//! - `graph_query` returns camelCase JSON (`"updatedAt"`, `"evidenceCount"`, -//! `"documentIds"`), which would need a hand-written camel→snake reader to -//! become a [`GraphRelationRecord`] again — that is new logic, and lossy. -//! -//! So kv reads and relation reads go through `MemoryClient::kv_records` / -//! `graph_relations`, thin `pub(crate)` forwarders onto the storage layer's -//! already-typed `kv_records_*` / `graph_relations_*`. Writes go through the -//! public `kv_set` / `graph_upsert`. -//! -//! ## `kv_get` is O(slice), knowingly -//! -//! There is no single-record getter anywhere below this line: the vendored -//! `KvStore` exposes `records_namespace` / `records_global` and nothing -//! narrower. Rather than add a fourth key-transform path and a new SELECT here, -//! this reads the slice and picks the key. The right fix is a -//! `record_namespace(ns, key)` / `record_global(key)` pair **upstream in the -//! vendored `KvStore`**, not a re-implementation in the driver. -//! -//! ## Two behaviours inherited from the storage layer, not introduced here -//! -//! - **Entities and predicates are upper-cased on write** by -//! `normalize_graph_entity` / `normalize_graph_predicate`, so a -//! `put_relation("Alice", "owns", "Phoenix")` reads back as -//! `("ALICE", "OWNS", "PHOENIX")`. Pinned by the storage layer's own tests; -//! the driver must not "fix" it. -//! - **`relations` cannot return more than 300 rows per underlying statement** — -//! every `graph_relations_*` SQL statement carries a hard-coded `LIMIT 300`. -//! A contract `limit` above that is silently unreachable. `limit` truncates -//! downward only. -//! -//! ## `updated_at` is not forwarded on write -//! -//! `graph_upsert_internal` stamps its own `now_ts()`. A driver must not let a -//! caller backdate a write, so [`GraphRelationRecord::updated_at`] is dropped -//! on the way in and re-read on the way out. - -use async_trait::async_trait; -use serde_json::{json, Value}; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::MemoryGraph; -use tinycortex_api::types::{GraphRelationRecord, MemoryKvRecord}; - -use super::{host_error, EmbeddedMemoryProvider}; - -/// The per-statement row ceiling every `graph_relations_*` query carries. -pub(super) const RELATION_ROW_CEILING: usize = 300; - -/// Rebuilds the attrs object the storage layer's `merge_graph_attrs` reads. -/// -/// The record's structured fields (`evidence_count`, `document_ids`, -/// `chunk_ids`, `order_index`) live *inside* `attrs` by the host's merge -/// convention, so dropping them would silently lose the caller's evidence. -fn attrs_for_upsert(relation: &GraphRelationRecord) -> Value { - let mut attrs = relation.attrs.as_object().cloned().unwrap_or_default(); - attrs.insert("evidence_count".to_string(), json!(relation.evidence_count)); - if !relation.document_ids.is_empty() { - attrs.insert("document_ids".to_string(), json!(relation.document_ids)); - } - if !relation.chunk_ids.is_empty() { - attrs.insert("chunk_ids".to_string(), json!(relation.chunk_ids)); - } - if let Some(order_index) = relation.order_index { - attrs.insert("order_index".to_string(), json!(order_index)); - } - Value::Object(attrs) -} - -#[async_trait] -impl MemoryGraph for EmbeddedMemoryProvider { - async fn kv_get( - &self, - namespace: Option<&str>, - key: &str, - ) -> Result, MemoryError> { - log::debug!( - "[memory:driver:embedded] kv_get namespace={} key_chars={}", - namespace.unwrap_or("-"), - key.chars().count() - ); - // The stored key is canonicalized on write, so compare against the - // same transform rather than the raw argument. - let wanted = crate::openhuman::memory::store::safety::canonical_identifier(key); - let records = self - .client() - .await? - .kv_records(namespace) - .await - .map_err(|error| host_error("kv_get", error))?; - Ok(records.into_iter().find(|record| record.key == wanted)) - } - - async fn kv_put( - &self, - namespace: Option<&str>, - key: &str, - value: Value, - ) -> Result<(), MemoryError> { - log::debug!( - "[memory:driver:embedded] kv_put namespace={} key_chars={}", - namespace.unwrap_or("-"), - key.chars().count() - ); - self.client() - .await? - .kv_set(namespace, key, &value) - .await - .map_err(|error| host_error("kv_put", error)) - } - - async fn kv_list( - &self, - namespace: Option<&str>, - prefix: Option<&str>, - limit: usize, - ) -> Result, MemoryError> { - log::debug!( - "[memory:driver:embedded] kv_list namespace={} prefix={} limit={limit}", - namespace.unwrap_or("-"), - prefix.unwrap_or("-") - ); - let mut records = self - .client() - .await? - .kv_records(namespace) - .await - .map_err(|error| host_error("kv_list", error))?; - if let Some(prefix) = prefix { - // Canonicalized for the same reason as `kv_get`: stored keys have - // already been through the transform. - let prefix = crate::openhuman::memory::store::safety::canonical_identifier(prefix); - records.retain(|record| record.key.starts_with(&prefix)); - } - records.truncate(limit); - Ok(records) - } - - async fn relations( - &self, - namespace: Option<&str>, - subject: Option<&str>, - predicate: Option<&str>, - limit: usize, - ) -> Result, MemoryError> { - log::debug!( - "[memory:driver:embedded] relations namespace={} subject={} predicate={} limit={limit}", - namespace.unwrap_or("-"), - subject.unwrap_or("-"), - predicate.unwrap_or("-") - ); - if limit > RELATION_ROW_CEILING { - log::debug!( - "[memory:driver:embedded] relations limit={limit} exceeds the storage ceiling \ - {RELATION_ROW_CEILING}; the query cannot return more" - ); - } - let mut rows = self - .client() - .await? - .graph_relations(namespace, subject, predicate) - .await - .map_err(|error| host_error("relations", error))?; - rows.truncate(limit); - Ok(rows) - } - - async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError> { - log::debug!( - "[memory:driver:embedded] put_relation namespace={} predicate={}", - relation.namespace.as_deref().unwrap_or("-"), - relation.predicate - ); - let attrs = attrs_for_upsert(&relation); - self.client() - .await? - .graph_upsert( - relation.namespace.as_deref(), - &relation.subject, - &relation.predicate, - &relation.object, - &attrs, - ) - .await - .map_err(|error| host_error("put_relation", error)) - } -} - -#[cfg(test)] -#[path = "graph_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/graph_tests.rs b/src/openhuman/memory/driver/embedded/graph_tests.rs deleted file mode 100644 index a6675a2285..0000000000 --- a/src/openhuman/memory/driver/embedded/graph_tests.rs +++ /dev/null @@ -1,300 +0,0 @@ -//! [`MemoryGraph`] tests. -//! -//! The load-bearing ones: -//! -//! - `kv_get_returns_a_record_with_a_timestamp` is why this family cannot -//! delegate to `MemoryClient::kv_get`, which returns a bare `Value` and drops -//! `updated_at`. -//! - `put_relation_round_trips_with_normalized_entities` pins the storage -//! layer's upper-casing. It is inherited behaviour, asserted so nobody -//! "fixes" it in the driver. -//! - the two `*_is_visible_to_*` tests are the same-store proofs. - -use super::super::test_support::fresh_driver; -use super::*; - -use serde_json::json; - -fn relation( - namespace: Option<&str>, - subject: &str, - predicate: &str, - object: &str, -) -> GraphRelationRecord { - GraphRelationRecord { - namespace: namespace.map(str::to_string), - subject: subject.to_string(), - predicate: predicate.to_string(), - object: object.to_string(), - attrs: json!({"note": "from the contract"}), - updated_at: 0.0, - evidence_count: 1, - order_index: None, - document_ids: vec!["doc-1".to_string()], - chunk_ids: vec![], - } -} - -#[tokio::test] -async fn kv_put_then_kv_get_returns_a_record_with_a_timestamp() { - let (_tmp, provider) = fresh_driver(); - - provider - .kv_put(Some("kv_ns"), "theme", json!("dark")) - .await - .expect("kv_put"); - - let record = provider - .kv_get(Some("kv_ns"), "theme") - .await - .expect("kv_get") - .expect("record exists"); - - assert_eq!(record.key, "theme"); - assert_eq!(record.value, json!("dark")); - assert_eq!(record.namespace.as_deref(), Some("kv_ns")); - assert!( - record.updated_at > 0.0, - "the contract's record carries updated_at; the bare-value path cannot" - ); -} - -#[tokio::test] -async fn kv_get_returns_none_for_unknown_key() { - let (_tmp, provider) = fresh_driver(); - assert!(provider - .kv_get(Some("kv_ns"), "absent") - .await - .expect("kv_get") - .is_none()); -} - -#[tokio::test] -async fn kv_put_with_none_namespace_writes_the_global_slice() { - let (_tmp, provider) = fresh_driver(); - - provider - .kv_put(None, "global_key", json!(7)) - .await - .expect("kv_put"); - - let record = provider - .kv_get(None, "global_key") - .await - .expect("kv_get") - .expect("record exists"); - assert_eq!(record.value, json!(7)); - assert!( - record.namespace.is_none(), - "a global row must report no namespace" - ); - - // And it must not leak into a namespace slice. - assert!(provider - .kv_get(Some("kv_ns"), "global_key") - .await - .expect("kv_get") - .is_none()); -} - -#[tokio::test] -async fn kv_list_applies_prefix_and_limit() { - let (_tmp, provider) = fresh_driver(); - for key in ["ui.theme", "ui.density", "net.proxy"] { - provider - .kv_put(Some("kv_ns"), key, json!(key)) - .await - .expect("kv_put"); - } - - let all = provider - .kv_list(Some("kv_ns"), None, 100) - .await - .expect("kv_list"); - assert_eq!(all.len(), 3); - - let ui = provider - .kv_list(Some("kv_ns"), Some("ui."), 100) - .await - .expect("kv_list"); - assert_eq!(ui.len(), 2, "prefix must narrow the slice: {ui:?}"); - assert!(ui.iter().all(|record| record.key.starts_with("ui."))); - - let capped = provider - .kv_list(Some("kv_ns"), None, 1) - .await - .expect("kv_list"); - assert_eq!(capped.len(), 1, "limit must truncate"); -} - -#[tokio::test] -async fn kv_list_with_none_namespace_reads_the_global_slice() { - let (_tmp, provider) = fresh_driver(); - provider - .kv_put(None, "g1", json!(1)) - .await - .expect("kv_put global"); - provider - .kv_put(Some("kv_ns"), "n1", json!(2)) - .await - .expect("kv_put namespaced"); - - let global = provider.kv_list(None, None, 100).await.expect("kv_list"); - let keys: Vec<&str> = global.iter().map(|record| record.key.as_str()).collect(); - assert!(keys.contains(&"g1")); - assert!( - !keys.contains(&"n1"), - "the global slice must not include namespaced rows: {keys:?}" - ); -} - -#[tokio::test] -async fn put_relation_round_trips_with_normalized_entities() { - let (_tmp, provider) = fresh_driver(); - - provider - .put_relation(relation(Some("g_ns"), "Alice", "owns", "Phoenix")) - .await - .expect("put_relation"); - - let rows = provider - .relations(Some("g_ns"), None, None, 50) - .await - .expect("relations"); - assert_eq!(rows.len(), 1); - let row = &rows[0]; - // Inherited from `normalize_graph_entity` / `normalize_graph_predicate`. - assert_eq!(row.subject, "ALICE"); - assert_eq!(row.predicate, "OWNS"); - assert_eq!(row.object, "PHOENIX"); - assert_eq!(row.namespace.as_deref(), Some("g_ns")); - // The structured fields survive the attrs round-trip. - assert_eq!(row.document_ids, vec!["doc-1".to_string()]); - assert!(row.evidence_count >= 1); - assert_eq!(row.attrs.get("note"), Some(&json!("from the contract"))); - assert!( - row.updated_at > 0.0, - "the store stamps its own updated_at; the caller's 0.0 must not survive" - ); -} - -#[tokio::test] -async fn relations_filters_by_subject_and_predicate() { - let (_tmp, provider) = fresh_driver(); - provider - .put_relation(relation(Some("g_ns"), "alice", "owns", "phoenix")) - .await - .expect("put_relation"); - provider - .put_relation(relation(Some("g_ns"), "alice", "likes", "tea")) - .await - .expect("put_relation"); - provider - .put_relation(relation(Some("g_ns"), "bob", "owns", "kettle")) - .await - .expect("put_relation"); - - let alice = provider - .relations(Some("g_ns"), Some("alice"), None, 50) - .await - .expect("relations"); - assert_eq!(alice.len(), 2, "{alice:?}"); - - let owns = provider - .relations(Some("g_ns"), None, Some("owns"), 50) - .await - .expect("relations"); - assert_eq!(owns.len(), 2, "{owns:?}"); - - let both = provider - .relations(Some("g_ns"), Some("alice"), Some("owns"), 50) - .await - .expect("relations"); - assert_eq!(both.len(), 1); - assert_eq!(both[0].object, "PHOENIX"); -} - -#[tokio::test] -async fn relations_truncates_to_the_limit() { - let (_tmp, provider) = fresh_driver(); - for object in ["one", "two", "three"] { - provider - .put_relation(relation(Some("g_ns"), "alice", "owns", object)) - .await - .expect("put_relation"); - } - let rows = provider - .relations(Some("g_ns"), None, None, 2) - .await - .expect("relations"); - assert_eq!(rows.len(), 2); -} - -#[tokio::test] -async fn relations_with_none_namespace_spans_namespaces_and_global() { - let (_tmp, provider) = fresh_driver(); - provider - .put_relation(relation(Some("ns_one"), "alice", "owns", "phoenix")) - .await - .expect("put_relation namespaced"); - provider - .put_relation(relation(None, "bob", "owns", "kettle")) - .await - .expect("put_relation global"); - - let all = provider - .relations(None, None, None, 100) - .await - .expect("relations"); - let subjects: Vec<&str> = all.iter().map(|row| row.subject.as_str()).collect(); - assert!(subjects.contains(&"ALICE"), "{subjects:?}"); - assert!(subjects.contains(&"BOB"), "{subjects:?}"); - - // The namespaced row must still be scoped, not global. - let global_only: Vec<&str> = all - .iter() - .filter(|row| row.namespace.is_none()) - .map(|row| row.subject.as_str()) - .collect(); - assert_eq!(global_only, vec!["BOB"]); -} - -#[tokio::test] -async fn kv_put_through_the_contract_is_visible_to_memory_client_kv_get() { - let (_tmp, provider) = fresh_driver(); - provider - .kv_put(Some("kv_ns"), "theme", json!("dark")) - .await - .expect("kv_put"); - - let via_client = provider - .client() - .await - .expect("client") - .kv_get(Some("kv_ns"), "theme") - .await - .expect("kv_get"); - assert_eq!(via_client, Some(json!("dark"))); -} - -#[tokio::test] -async fn put_relation_through_the_contract_is_visible_to_memory_client_graph_query() { - let (_tmp, provider) = fresh_driver(); - provider - .put_relation(relation(Some("g_ns"), "alice", "owns", "phoenix")) - .await - .expect("put_relation"); - - let via_client = provider - .client() - .await - .expect("client") - .graph_query(Some("g_ns"), None, None) - .await - .expect("graph_query"); - assert_eq!(via_client.len(), 1, "{via_client:?}"); - assert_eq!( - via_client[0].get("subject").and_then(|v| v.as_str()), - Some("ALICE") - ); -} diff --git a/src/openhuman/memory/driver/embedded/ingest.rs b/src/openhuman/memory/driver/embedded/ingest.rs deleted file mode 100644 index 74e0aa3930..0000000000 --- a/src/openhuman/memory/driver/embedded/ingest.rs +++ /dev/null @@ -1,217 +0,0 @@ -//! [`MemoryIngest`] for the embedded driver — bulk content into the chunk -//! tier. -//! -//! Both methods re-shape an [`IngestItem`] batch into the canonicaliser input -//! `memory::ingest_pipeline` already takes and hand it straight over. No -//! chunking, splitting, scoring, or dedupe decision happens in this file; all -//! of it is the engine's. -//! -//! ## TAINT — this driver REFUSES what it cannot persist -//! -//! [`IngestItem::taint`] is a host-stamped provenance marker that a driver -//! "must persist what it is given and must never assign or upgrade". The chunk -//! tier **cannot** carry it: `Chunk::metadata` has no taint field, and neither -//! `ingest_document_versioned` nor the engine's `ingest::pipeline` takes a -//! taint parameter. Taint lives on the *other* tier — `MemoryTaint` is a column -//! on the `UnifiedMemory` namespace-document path reached through -//! `Memory::store_with_taint`, which is `MemoryCore::store`'s (M3a) business, -//! not this family's. -//! -//! Three ways to handle that, and only one is defensible: -//! -//! 1. **Refuse** a non-default taint with [`MemoryError::Invalid`] — what this -//! file does. A driver that must persist what it is given and cannot must -//! not accept the call. -//! 2. Smuggle it into `tags` as a reserved label. That invents an on-disk -//! convention, which this step is explicitly not allowed to do. -//! 3. Drop it silently. This is the failure mode the rule exists to prevent: -//! externally-synced content would land indistinguishable from user-authored -//! content, which is a prompt-injection trust boundary, not a formatting -//! detail. -//! -//! `ingest_refuses_non_default_taint` is the security test that pins this. If a -//! future change gives the chunk tier a taint column, replace the refusal with -//! a real write — never with a drop. -//! -//! ## Fields with nowhere to go, stated rather than dropped -//! -//! - **`namespace`** — the chunk tier is keyed by `(source_kind, source_id)` -//! and has no namespace column. Ignored. -//! - **`mime`** — the canonicaliser takes decoded text only. A text-ish MIME is -//! accepted and dropped; anything else is [`MemoryError::Invalid`], which is -//! the case the contract names. -//! - **`title`** — [`IngestItem`] has none, so `DocumentInput.title` is empty. -//! `document::canonicalise` only bails when title *and* body are empty, so a -//! body-only document still ingests. -//! -//! ## `skipped` reports dedupe, not drops -//! -//! `IngestSummary` has two "not written" signals: `already_ingested` (the -//! whole call was a dedupe no-op) and `chunks_dropped` (the fast-score path -//! rejected individual chunks). The contract's `skipped` is "units the driver -//! recognised as already present", so `already_ingested` wins when set and -//! `chunks_dropped` fills in otherwise. They are never summed — that would -//! double-count a number the caller uses to detect a silently-dropping driver. - -use async_trait::async_trait; -use chrono::Utc; -use tinycortex::memory::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; -use tinycortex::memory::ingest::canonicalize::document::DocumentInput; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::{IngestItem, IngestOutcome}; -use tinycortex_api::provider::MemoryIngest; -use tinycortex_api::types::MemoryTaint; - -use crate::openhuman::memory::ingest_pipeline::{self, IngestResult}; - -use super::{host_error, EmbeddedMemoryProvider}; - -/// MIME types the canonicaliser's "already decoded to text" contract covers. -/// -/// Deliberately a prefix/suffix rule rather than an exhaustive list: every -/// `text/*` type is text by definition, and the structured-text families -/// (`+json`, `+xml`) decode to text too. Anything else — a PDF, an image, an -/// archive — is content this path cannot honestly ingest as a string. -fn is_text_mime(mime: &str) -> bool { - let mime = mime.trim().to_ascii_lowercase(); - let base = mime.split(';').next().unwrap_or("").trim().to_string(); - base.starts_with("text/") - || base.ends_with("+json") - || base.ends_with("+xml") - || matches!( - base.as_str(), - "application/json" | "application/xml" | "application/x-ndjson" - ) -} - -/// The checks every item must pass regardless of which method received it. -fn validate(item: &IngestItem) -> Result<(), MemoryError> { - if item.taint != MemoryTaint::default() { - // See the module docs. This is a refusal, not a limitation to route - // around. - return Err(MemoryError::Invalid(format!( - "ingest cannot preserve taint '{}': the chunk tier has no taint column, and a \ - driver must never silently downgrade provenance", - item.taint.as_db_str() - ))); - } - if let Some(mime) = item.mime.as_deref() { - if !is_text_mime(mime) { - return Err(MemoryError::Invalid(format!( - "unsupported MIME '{mime}': ingest accepts decoded text only" - ))); - } - } - if item.content.trim().is_empty() { - return Err(MemoryError::Invalid( - "ingest content must not be empty".to_string(), - )); - } - Ok(()) -} - -/// Maps the engine's ingest summary onto the contract's outcome. -fn to_outcome(result: IngestResult) -> IngestOutcome { - IngestOutcome { - written: u32::try_from(result.chunks_written).unwrap_or(u32::MAX), - skipped: if result.already_ingested { - 1 - } else { - u32::try_from(result.chunks_dropped).unwrap_or(u32::MAX) - }, - ids: result.chunk_ids, - } -} - -#[async_trait] -impl MemoryIngest for EmbeddedMemoryProvider { - async fn ingest_document(&self, item: IngestItem) -> Result { - log::debug!( - "[memory:driver:embedded] ingest_document source={} source_id={} content_chars={}", - item.source.as_str(), - item.source_id, - item.content.chars().count() - ); - validate(&item)?; - - let doc = DocumentInput { - provider: item.source.as_str().to_string(), - // `IngestItem` carries no title; see the module docs. - title: String::new(), - body: item.content, - modified_at: item.timestamp.unwrap_or_else(Utc::now), - source_ref: item.source_ref.map(|source_ref| source_ref.value), - }; - - let config = self.config().await?; - ingest_pipeline::ingest_document_with_scope( - config, - &item.source_id, - &item.owner, - item.tags, - doc, - item.path_scope, - ) - .await - .map(to_outcome) - .map_err(|error| host_error("ingest_document", format!("{error:#}"))) - } - - async fn ingest_chat(&self, messages: Vec) -> Result { - log::debug!( - "[memory:driver:embedded] ingest_chat items={}", - messages.len() - ); - // The canonicaliser treats an empty batch as nothing to ingest, so - // this short-circuit changes no behaviour — it just avoids reading the - // config and touching the store to do nothing. - let Some(first) = messages.first() else { - return Ok(IngestOutcome::default()); - }; - - let source_id = first.source_id.clone(); - let owner = first.owner.clone(); - let tags = first.tags.clone(); - let platform = first.source.as_str().to_string(); - - for item in &messages { - validate(item)?; - // The contract says the batch shares one conversation and that - // ordering within it is significant. A batch spanning two sources - // would be silently attributed to the first one's `source_id`, - // which is the dedupe key — so refuse instead of guessing. - if item.source_id != source_id { - return Err(MemoryError::Invalid(format!( - "ingest_chat batch mixes source ids ('{source_id}' and '{}'); one batch is \ - one conversation", - item.source_id - ))); - } - } - - let batch = ChatBatch { - platform, - channel_label: source_id.clone(), - messages: messages - .into_iter() - .map(|item| ChatMessage { - // The only author-ish field `IngestItem` has. - author: item.owner, - timestamp: item.timestamp.unwrap_or_else(Utc::now), - text: item.content, - source_ref: item.source_ref.map(|source_ref| source_ref.value), - }) - .collect(), - }; - - let config = self.config().await?; - ingest_pipeline::ingest_chat(config, &source_id, &owner, tags, batch) - .await - .map(to_outcome) - .map_err(|error| host_error("ingest_chat", format!("{error:#}"))) - } -} - -#[cfg(test)] -#[path = "ingest_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/ingest_tests.rs b/src/openhuman/memory/driver/embedded/ingest_tests.rs deleted file mode 100644 index 184b183c3f..0000000000 --- a/src/openhuman/memory/driver/embedded/ingest_tests.rs +++ /dev/null @@ -1,219 +0,0 @@ -//! [`MemoryIngest`] tests for the embedded driver. -//! -//! `ingest_refuses_non_default_taint` is the security test: it pins that a -//! taint the chunk tier cannot carry is *refused*, never silently dropped. - -use super::super::test_support::fresh_driver; - -use chrono::{TimeZone, Utc}; -use tinycortex_api::chunks::{DataSource, SourceRef}; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::IngestItem; -use tinycortex_api::provider::MemoryIngest; -use tinycortex_api::types::MemoryTaint; - -const BASE_MS: i64 = 1_700_000_000_000; - -fn item(source: DataSource, source_id: &str, content: &str, offset_ms: i64) -> IngestItem { - IngestItem { - namespace: None, - source, - source_id: source_id.to_string(), - owner: "alice".to_string(), - source_ref: Some(SourceRef::new(format!("{}://x", source.as_str()))), - content: content.to_string(), - mime: None, - timestamp: Some(Utc.timestamp_millis_opt(BASE_MS + offset_ms).unwrap()), - tags: Vec::new(), - taint: MemoryTaint::default(), - path_scope: None, - } -} - -// ── documents ──────────────────────────────────────────────────────────── - -#[tokio::test] -async fn ingest_document_writes_chunks_and_reports_counts() { - let (_tmp, provider) = fresh_driver(); - let outcome = provider - .ingest_document(item( - DataSource::Notion, - "doc-phoenix", - "The Phoenix migration launch window is Friday at 22:00 UTC.", - 0, - )) - .await - .expect("ingest_document"); - - assert!(outcome.written >= 1, "at least one chunk written"); - assert_eq!( - outcome.ids.len(), - outcome.written as usize, - "ids must line up with the written count" - ); -} - -#[tokio::test] -async fn ingest_document_rejects_empty_body_as_invalid() { - let (_tmp, provider) = fresh_driver(); - let error = provider - .ingest_document(item(DataSource::Notion, "doc-empty", " \n ", 0)) - .await - .expect_err("an empty body must be refused"); - assert!( - matches!(error, MemoryError::Invalid(_)), - "expected Invalid, got {error:?}" - ); -} - -#[tokio::test] -async fn ingest_document_rejects_binary_mime_as_invalid() { - let (_tmp, provider) = fresh_driver(); - let mut doc = item(DataSource::Notion, "doc-pdf", "%PDF-1.7", 0); - doc.mime = Some("application/pdf".to_string()); - - let error = provider - .ingest_document(doc) - .await - .expect_err("a non-text MIME must be refused"); - assert!( - matches!(error, MemoryError::Invalid(_)), - "expected Invalid, got {error:?}" - ); -} - -#[tokio::test] -async fn ingest_document_accepts_text_mime() { - let (_tmp, provider) = fresh_driver(); - let mut doc = item(DataSource::Notion, "doc-md", "# Phoenix\n\nlaunch notes", 0); - doc.mime = Some("text/markdown; charset=utf-8".to_string()); - - provider - .ingest_document(doc) - .await - .expect("text/* must be accepted"); -} - -// ── the taint refusal ──────────────────────────────────────────────────── - -#[tokio::test] -async fn ingest_refuses_non_default_taint() { - let (_tmp, provider) = fresh_driver(); - - let mut doc = item(DataSource::Notion, "doc-external", "synced body", 0); - doc.taint = MemoryTaint::ExternalSync; - let error = provider - .ingest_document(doc) - .await - .expect_err("the chunk tier cannot carry taint, so the call must be refused"); - match &error { - MemoryError::Invalid(reason) => assert!( - reason.contains("taint"), - "the refusal must name taint so an operator can act on it, got {reason}" - ), - other => panic!("expected Invalid, got {other:?}"), - } - - let mut chat = item(DataSource::Telegram, "chan-1", "hello", 0); - chat.taint = MemoryTaint::ExternalSync; - let error = provider - .ingest_chat(vec![chat]) - .await - .expect_err("the chat path must refuse identically"); - assert!( - matches!(error, MemoryError::Invalid(_)), - "expected Invalid, got {error:?}" - ); -} - -// ── chat ───────────────────────────────────────────────────────────────── - -#[tokio::test] -async fn ingest_chat_empty_batch_is_a_successful_noop() { - let (_tmp, provider) = fresh_driver(); - let outcome = provider.ingest_chat(Vec::new()).await.expect("empty batch"); - assert_eq!(outcome.written, 0); - assert_eq!(outcome.skipped, 0); - assert!(outcome.ids.is_empty()); -} - -#[tokio::test] -async fn ingest_chat_writes_the_whole_conversation() { - let (_tmp, provider) = fresh_driver(); - let outcome = provider - .ingest_chat(vec![ - item(DataSource::Telegram, "chan-1", "phoenix ships friday", 0), - item( - DataSource::Telegram, - "chan-1", - "confirmed, 22:00 UTC", - 1_000, - ), - ]) - .await - .expect("ingest_chat"); - assert!(outcome.written >= 1); -} - -#[tokio::test] -async fn ingest_chat_preserves_message_order() { - let (_tmp, provider) = fresh_driver(); - provider - .ingest_chat(vec![ - item(DataSource::Telegram, "chan-order", "first message", 0), - item(DataSource::Telegram, "chan-order", "second message", 1_000), - ]) - .await - .expect("ingest_chat"); - - let config = provider.config().await.expect("config"); - let chunks = crate::openhuman::memory::store::chunks::store::list_chunks( - config, - &crate::openhuman::memory::store::chunks::store::ListChunksQuery { - source_id: Some("chan-order".to_string()), - limit: Some(50), - ..Default::default() - }, - ) - .expect("list_chunks"); - - let body = chunks - .iter() - .map(|chunk| chunk.content.as_str()) - .collect::>() - .join("\n"); - let first = body.find("first message"); - let second = body.find("second message"); - match (first, second) { - (Some(first), Some(second)) => assert!( - first < second, - "chronological order must survive canonicalisation" - ), - // The chunker may split per message; then ordering is asserted by - // sequence instead. - _ => { - let mut seqs = chunks.iter().map(|c| c.seq_in_source).collect::>(); - seqs.sort_unstable(); - assert!(!seqs.is_empty(), "the batch must have produced chunks"); - } - } -} - -#[tokio::test] -async fn ingest_chat_rejects_mixed_source_ids_as_invalid() { - let (_tmp, provider) = fresh_driver(); - let error = provider - .ingest_chat(vec![ - item(DataSource::Telegram, "chan-1", "hello", 0), - item(DataSource::Telegram, "chan-2", "different channel", 1_000), - ]) - .await - .expect_err("one batch is one conversation"); - match error { - MemoryError::Invalid(reason) => assert!( - reason.contains("source id"), - "the refusal must explain itself, got {reason}" - ), - other => panic!("expected Invalid, got {other:?}"), - } -} diff --git a/src/openhuman/memory/driver/embedded/maintenance.rs b/src/openhuman/memory/driver/embedded/maintenance.rs deleted file mode 100644 index 0a83c005e8..0000000000 --- a/src/openhuman/memory/driver/embedded/maintenance.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! [`MemoryMaintenance`] for the embedded driver — the four upkeep operations -//! the host's scheduler drives. -//! -//! No operation here installs a background task, and none of them is allowed to -//! return a success-shaped empty report for work that did not happen. Where the -//! embedded engine has no mechanism behind a contract operation, the report says -//! so in [`MaintenanceReport::findings`] rather than reading as "ran, nothing to -//! do". -//! -//! ## `reembed` and `consolidate` enqueue; they do not run -//! -//! Both go through the job queue, which is how the host itself drives them. The -//! reported `changed` is therefore *jobs enqueued*, and each report says -//! explicitly that the work runs asynchronously in the queue worker. That is -//! also why [`MemoryError::BudgetExceeded`] never appears here despite the -//! contract naming it for `reembed`: the embedding budget is exhausted inside -//! the worker, long after this call has returned. -//! -//! `consolidate` in particular must go through the queue rather than through -//! `tree::tree::flush::flush_stale_buffers_default`. The queue path fans out -//! into per-tree `Seal` jobs whose label strategy the worker derives per tree -//! (`TreeFactory::from_tree(&tree).label_strategy(...)`); the direct call takes -//! a single `LabelStrategy` for every tree, has no production caller, and would -//! apply one tree kind's labelling to all of them. -//! -//! ## `compact` under-delivers against the contract, and says so -//! -//! The contract asks for "vacuum indexes, drop tombstones, prune dead -//! references". **There is no `VACUUM` anywhere in this tree** — not in -//! `src/openhuman/memory/`, not in the vendored engine. The only thing the -//! embedded engine has that is genuinely in this family is -//! `queue::store::recover_stale_locks`, which drops lock rows referencing -//! workers that are gone: dead references, prunable. So that is what runs, and -//! the findings state plainly that no index vacuum exists. -//! -//! Two things deliberately do **not** run under `compact`: -//! `requeue_failed` / `requeue_transient_failed` are liveness self-heal, not -//! space reclamation, and the periodic scheduler already drives the transient -//! one every three hours — doing it again under a name that means something -//! else would make the operation lie in a second way. `diff::ops::cleanup` does -//! reclaim (checkpoint tags), but needs a retention window the contract does not -//! supply, and inventing one here would be policy. -//! -//! ## `doctor` is read-only and reports `note`, not `failure` -//! -//! `changed` is hard-coded `0`; the contract requires it. Findings come from -//! [`StageHealth::note`], documented as "short non-localized human note for logs -//! / CLI (never a secret)" — exactly the constraint `findings` imposes. -//! `PipelineFailure` carries an i18n remediation key meant for the UI, and -//! `DegradedState::cause` has no such never-a-secret guarantee, so neither is -//! used. - -use async_trait::async_trait; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::MaintenanceReport; -use tinycortex_api::provider::MemoryMaintenance; - -use crate::openhuman::config::Config; -use crate::openhuman::memory::queue::store as queue_store; -use crate::openhuman::memory::queue::types::JobStatus; - -use super::{host_error, EmbeddedMemoryProvider}; - -/// Total jobs plus ready jobs, both best-effort. -/// -/// A counter read that errors degrades to 0 rather than failing the whole -/// operation, matching `tree::health::doctor`'s own rule for the same counters — -/// maintenance reporting is a convenience, not an audit. -fn queue_counts(config: &Config) -> (u64, u64) { - let total = queue_store::count_total(config).unwrap_or(0); - let ready = queue_store::count_by_status(config, JobStatus::Ready).unwrap_or(0); - (total, ready) -} - -#[async_trait] -impl MemoryMaintenance for EmbeddedMemoryProvider { - async fn reembed(&self) -> Result { - log::debug!("[memory:driver:embedded] reembed"); - let config = self.config().await?.clone(); - - // `ensure_reembed_backfill` opens SQLite and is synchronous; it also - // swallows its own errors by design (it must never fail a settings - // save), so the observable effect is the queue depth delta below. - let (examined, enqueued) = tokio::task::spawn_blocking(move || { - let (total, ready_before) = queue_counts(&config); - crate::openhuman::memory::queue::ensure_reembed_backfill(&config); - let (_, ready_after) = queue_counts(&config); - (total, ready_after.saturating_sub(ready_before)) - }) - .await - .map_err(|error| host_error("reembed", format!("join: {error}")))?; - - Ok(MaintenanceReport { - operation: "reembed".to_string(), - examined, - changed: enqueued, - findings: vec![format!( - "enqueued {enqueued} re-embed backfill job(s); the work itself runs \ - asynchronously in the memory queue worker" - )], - }) - } - - async fn compact(&self) -> Result { - log::debug!("[memory:driver:embedded] compact"); - let config = self.config().await?.clone(); - - let (examined, recovered) = tokio::task::spawn_blocking(move || { - let (total, _ready) = queue_counts(&config); - let recovered = queue_store::recover_stale_locks(&config).unwrap_or(0); - (total, recovered as u64) - }) - .await - .map_err(|error| host_error("compact", format!("join: {error}")))?; - - Ok(MaintenanceReport { - operation: "compact".to_string(), - examined, - changed: recovered, - findings: vec![ - format!("released {recovered} stale queue lock(s)"), - "no index vacuum or tombstone pruning is implemented by the embedded engine; \ - compact reclaims dead queue locks only" - .to_string(), - ], - }) - } - - async fn consolidate(&self) -> Result { - log::debug!("[memory:driver:embedded] consolidate"); - let config = self.config().await?.clone(); - - let (examined, enqueued) = - tokio::task::spawn_blocking(move || -> Result<(u64, bool), String> { - let (total, _ready) = queue_counts(&config); - let enqueued = - crate::openhuman::memory::queue::scheduler::enqueue_flush_stale_job(&config)?; - Ok((total, enqueued)) - }) - .await - .map_err(|error| host_error("consolidate", format!("join: {error}")))? - .map_err(|error| host_error("consolidate", error))?; - - let finding = if enqueued { - "enqueued a stale-buffer flush; it fans out into per-tree seal jobs in the memory \ - queue worker" - } else { - // Not a failure: the enqueue is deduped on (date, 3-hour block), so - // a second call inside the same window is a genuine no-op and must - // not report work it did not do. - "a stale-buffer flush is already queued for this window; nothing was enqueued" - }; - Ok(MaintenanceReport { - operation: "consolidate".to_string(), - examined, - changed: u64::from(enqueued), - findings: vec![finding.to_string()], - }) - } - - async fn doctor(&self) -> Result { - log::debug!("[memory:driver:embedded] doctor"); - let config = self.config().await?; - - // Infallible and already `spawn_blocking`-wrapped host-side. - let report = crate::openhuman::memory::tree::health::doctor::async_run_doctor(config).await; - - let findings = report - .stages - .iter() - .filter(|stage| !stage.ok) - .map(|stage| format!("{}: {}", stage.stage, stage.note)) - .collect::>(); - - Ok(MaintenanceReport { - operation: "doctor".to_string(), - examined: report.counters.total_chunks, - // Read-only by contract. Not derived from anything — pinned. - changed: 0, - findings, - }) - } -} - -#[cfg(test)] -#[path = "maintenance_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/maintenance_tests.rs b/src/openhuman/memory/driver/embedded/maintenance_tests.rs deleted file mode 100644 index f19d383798..0000000000 --- a/src/openhuman/memory/driver/embedded/maintenance_tests.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! [`MemoryMaintenance`] tests. -//! -//! The point of most of these is *honesty*, not throughput: each operation must -//! either do the work it names or say in `findings` that it did not. A report -//! with `changed: 0` and an empty `findings` list reads as "ran, nothing to do" -//! and is exactly what these tests exist to prevent. - -use super::super::test_support::fresh_driver; -use super::*; - -#[tokio::test] -async fn doctor_never_reports_changed_and_names_itself() { - let (_tmp, provider) = fresh_driver(); - let report = provider.doctor().await.expect("doctor"); - - assert_eq!(report.operation, "doctor"); - assert_eq!( - report.changed, 0, - "the contract requires doctor to be read-only" - ); -} - -#[tokio::test] -async fn doctor_is_repeatable_and_stays_read_only() { - let (_tmp, provider) = fresh_driver(); - let first = provider.doctor().await.expect("first doctor"); - let second = provider.doctor().await.expect("second doctor"); - assert_eq!(first.changed, 0); - assert_eq!(second.changed, 0); - assert_eq!(first.examined, second.examined); -} - -#[tokio::test] -async fn reembed_reports_an_enqueue_rather_than_a_run() { - let (_tmp, provider) = fresh_driver(); - let report = provider.reembed().await.expect("reembed"); - - assert_eq!(report.operation, "reembed"); - assert!( - report - .findings - .iter() - .any(|f| f.contains("asynchronously") && f.contains("queue")), - "the report must not imply the re-embed already happened: {:?}", - report.findings - ); -} - -#[tokio::test] -async fn compact_states_that_no_vacuum_exists() { - let (_tmp, provider) = fresh_driver(); - let report = provider.compact().await.expect("compact"); - - assert_eq!(report.operation, "compact"); - assert!( - report - .findings - .iter() - .any(|f| f.contains("no index vacuum")), - "compact must not silently under-deliver against the contract: {:?}", - report.findings - ); - assert!( - !report.findings.is_empty(), - "an empty findings list would read as 'ran, nothing to do'" - ); -} - -#[tokio::test] -async fn consolidate_enqueues_once_per_window_and_says_so_the_second_time() { - let (_tmp, provider) = fresh_driver(); - - let first = provider.consolidate().await.expect("first consolidate"); - assert_eq!(first.operation, "consolidate"); - assert_eq!(first.changed, 1, "the first call enqueues a flush"); - assert!(first.findings.iter().any(|f| f.contains("enqueued"))); - - // Deduped on (date, 3-hour block). A second call inside the window really - // did nothing, and must not claim otherwise. - let second = provider.consolidate().await.expect("second consolidate"); - assert_eq!(second.changed, 0); - assert!( - second.findings.iter().any(|f| f.contains("already queued")), - "got: {:?}", - second.findings - ); -} - -#[tokio::test] -async fn every_operation_labels_itself_with_its_own_name() { - let (_tmp, provider) = fresh_driver(); - // A copy-paste `operation` string is the kind of thing only an explicit - // check catches — the reports are otherwise shaped identically. - assert_eq!( - provider.reembed().await.expect("reembed").operation, - "reembed" - ); - assert_eq!( - provider.compact().await.expect("compact").operation, - "compact" - ); - assert_eq!( - provider.consolidate().await.expect("consolidate").operation, - "consolidate" - ); - assert_eq!(provider.doctor().await.expect("doctor").operation, "doctor"); -} diff --git a/src/openhuman/memory/driver/embedded/mod.rs b/src/openhuman/memory/driver/embedded/mod.rs deleted file mode 100644 index 8946c3bc85..0000000000 --- a/src/openhuman/memory/driver/embedded/mod.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! The embedded `tinycortex` memory driver — the in-process engine behind the -//! [`MemoryProvider`] contract. -//! -//! This is the driver bound for [`DriverClass::Embedded`](crate::core::subsystem::DriverClass), -//! replacing the `NullMemoryProvider` placeholder M2b used to prove the binding -//! plumbing. -//! -//! ## It re-shapes, it does not re-implement -//! -//! Every method here delegates to an existing host call. There is no -//! retrieval, ranking, chunking, or storage logic in this directory — if a -//! change to a file under `driver/embedded/` starts to *decide* something about -//! memory rather than translate a call, it belongs in the engine instead. -//! -//! ## Construction is synchronous and does no I/O -//! -//! [`crate::openhuman::memory::binding::for_workspace`] is reached from -//! [`CoreContext::memory_binding`](crate::core::runtime::CoreContext::memory_binding), -//! which is documented as synchronous and I/O-free and is called from roughly -//! four thousand pre-boot unit tests with **no tokio runtime**. Meanwhile -//! [`MemoryClient::from_workspace_dir`](crate::openhuman::memory::store::MemoryClient::from_workspace_dir) -//! opens SQLite, runs migrations, and `tokio::spawn`s the ingestion worker — it -//! panics outside a runtime. -//! -//! So the driver holds a [`tokio::sync::OnceCell`] and resolves its client on -//! the first *async contract call*, not at bind time. [`Self::new`] touches -//! nothing on disk; `constructing_the_driver_does_no_io_and_needs_no_runtime` -//! pins that. -//! -//! ## Capability honesty -//! -//! [`MemoryProvider::capabilities`] must advertise only what is reachable — -//! [`tinycortex_api::provider::audit_provider`] compares the advertised set -//! against the `as_*` accessors and fails on either kind of disagreement. -//! [`advertised_capabilities`] and the `as_*` overrides therefore widen -//! together, once per M3 step: M3a landed the mandatory three, M3b documents / -//! graph / tool memory, M3c ingest / tree / entities, and M3d the last four — -//! diff, goals, sources and maintenance. The advertised set is now -//! [`Capabilities::all`], which is what the whole milestone existed to reach: a -//! bound context and an unbound one finally agree, so `memory_capabilities()` -//! becomes safe to gate on (M4). - -mod core_family; -#[cfg(feature = "memory-git")] -mod diff; -mod documents; -mod entities; -mod goals; -mod graph; -mod ingest; -mod maintenance; -mod portability; -mod recall; -mod sources; -mod tool_memory; -mod tree; - -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use async_trait::async_trait; -use tinycortex_api::capabilities::Capabilities; -#[cfg(not(feature = "memory-git"))] -use tinycortex_api::capabilities::Capability; -use tinycortex_api::error::MemoryError; -use tinycortex_api::health::MemoryHealth; -use tinycortex_api::provider::MemoryProvider; -use tokio::sync::OnceCell; - -use crate::openhuman::config::schema::MemoryHooksConfig; -use crate::openhuman::config::Config; -use crate::openhuman::memory::global; -use crate::openhuman::memory::store::MemoryClientRef; -use crate::openhuman::memory::Memory; - -/// The stable [`MemoryProvider::driver_id`] of this driver. -/// -/// Matches the `[subsystems.memory] driver` default (`default_memory_driver` -/// in `config::schema::subsystems`), so a default-configured host reports the -/// same id from config and from the bound provider. -/// -/// **Re-exported, not re-declared.** The same id is what -/// [`tinymemory::registry`] reserves at [`DriverClass::Embedded`], and -/// admission is decided there. Two independent string literals that must agree -/// is precisely the kind of pair that silently stops agreeing: the day one is -/// edited, `admit` would stop recognising this driver and every bind would fall -/// back to the null placeholder — loudly in the logs, but with memory writes -/// discarded for the whole run. One constant, one definition, no drift. -pub use tinymemory::registry::TINYCORTEX_DRIVER_ID as EMBEDDED_DRIVER_ID; - -/// The families this driver advertises: **all thirteen**. -/// -/// Written as [`Capabilities::all`] rather than a `mandatory().with(…)` chain -/// of thirteen terms, because the two are now the same value and the equality is -/// the point — `embedded_driver_advertises_every_capability` asserts exactly -/// that. `Capability` is deliberately not `#[non_exhaustive]`, so a fourteenth -/// family added to the contract widens `all()` here and fails -/// `audit_provider` until its accessor lands, which is the intended pressure. -fn advertised_capabilities() -> Capabilities { - // Without `memory-git` there is no git ledger, so the diff family has no - // implementation to reach. Dropping it here is not cosmetic: a provider - // that advertises a capability whose accessor returns `None` fails - // `audit_provider`, and callers are entitled to trust the advertised set - // rather than probing every accessor. - #[cfg(not(feature = "memory-git"))] - { - Capabilities::all().without(Capability::Diff) - } - #[cfg(feature = "memory-git")] - { - Capabilities::all() - } -} - -/// The in-process tinycortex driver for one workspace. -pub struct EmbeddedMemoryProvider { - workspace_dir: PathBuf, - /// Hook budgets from `[subsystems.memory.hooks]`. Carried so the - /// auto-recall / auto-capture guard (M4) has them without re-reading - /// config; no family in M3a consults them. - hooks: MemoryHooksConfig, - /// Resolved lazily — see the module docs for why this is not a plain - /// `MemoryClientRef`. - client: OnceCell, - /// Resolved lazily, for the same reason as [`Self::client`] — see - /// [`Self::config`]. - config: OnceCell, -} - -impl EmbeddedMemoryProvider { - /// Build a driver for `workspace_dir`. - /// - /// Synchronous, infallible, and I/O-free by contract: the workspace is not - /// created, SQLite is not opened, and no task is spawned until the first - /// async contract call. - pub fn new(workspace_dir: impl Into, hooks: MemoryHooksConfig) -> Self { - Self { - workspace_dir: workspace_dir.into(), - hooks, - client: OnceCell::new(), - config: OnceCell::new(), - } - } - - /// The workspace this driver is bound to. - pub fn workspace_dir(&self) -> &Path { - &self.workspace_dir - } - - /// The configured hook budgets. - pub fn hooks(&self) -> MemoryHooksConfig { - self.hooks - } - - /// The backing client, constructing it on first use. - /// - /// Goes through [`global::client_for_workspace`] rather than - /// `MemoryClient::from_workspace_dir` so a workspace that already has the - /// process-global client reuses it — two clients over one workspace means - /// two ingestion workers against the same SQLite file. - async fn client(&self) -> Result<&MemoryClientRef, MemoryError> { - self.client - .get_or_try_init(|| async { - global::client_for_workspace(&self.workspace_dir).map_err(|error| { - log::warn!( - "[memory:driver:embedded] workspace={} client init failed: {error}", - self.workspace_dir.display() - ); - MemoryError::Other(anyhow::anyhow!(error)) - }) - }) - .await - } - - /// The `Memory` handle every mandatory family delegates through. - pub(super) async fn memory(&self) -> Result, MemoryError> { - Ok(self.client().await?.memory_handle()) - } - - /// The host [`Config`] the chunk / tree / ingest layers are addressed - /// through, constructing it on first use. - /// - /// ## Why the driver needs a `Config` at all - /// - /// The families landed in M3a/M3b reach storage through `MemoryClient`, - /// which is rooted at a workspace directory and needs nothing else. The - /// three M3c families do not have that luxury: every entry point under - /// `memory::tree::tree_runtime`, `memory::store::chunks` and - /// `memory::ingest_pipeline` takes `&Config` and funnels it through - /// [`tinycortex::engine_config`](crate::openhuman::memory::tinycortex::engine_config), - /// which derives the engine's `MemoryConfig` — workspace root, embedding - /// model/dimensions, tree token budgets — from it. There is no - /// workspace-only door into those layers, and inventing one would be - /// engine logic. - /// - /// ## Why it is loaded, not defaulted - /// - /// `Config::default()` would silently substitute default embedding - /// dimensions and would report "no summarization provider" for a host that - /// has one configured. So the real config is loaded — the one belonging to - /// **this driver's** workspace, via - /// [`load_config_for_workspace_with_timeout`](crate::openhuman::config::load_config_for_workspace_with_timeout). - /// - /// Re-anchoring `workspace_dir` after a process-global load is not enough, - /// and that is what this used to do: everything *else* in the snapshot — - /// embedding routes, model dimensions, provider credentials, tree budgets — - /// would still be whichever workspace the process-global active-user / - /// `OPENHUMAN_WORKSPACE` resolution named at first use. A driver bound to B - /// would then run A's settings over B's files, sending data to the wrong - /// endpoint or writing an index at the wrong dimension. The loader resolves - /// the config file beside `self.workspace_dir` instead, and only falls back - /// to the process-global one when the workspace has no config of its own. - /// - /// Lazy for the same reason as [`Self::client`] — loading is async and - /// touches disk, and bind time is neither. - pub(super) async fn config(&self) -> Result<&Config, MemoryError> { - self.config - .get_or_try_init(|| async { - let config = crate::openhuman::config::load_config_for_workspace_with_timeout( - &self.workspace_dir, - ) - .await - .map_err(|error| { - log::warn!( - "[memory:driver:embedded] workspace={} config load failed: {error}", - self.workspace_dir.display() - ); - MemoryError::Other(anyhow::anyhow!("memory driver config load: {error}")) - })?; - debug_assert_eq!(config.workspace_dir, self.workspace_dir); - Ok(config) - }) - .await - } -} - -/// Maps an engine `anyhow` failure onto the contract's error type. -/// -/// The engine's [`Memory`] trait is deliberately `anyhow`-typed (it is an -/// internal storage abstraction with heterogeneous backends), so everything it -/// returns is opaque and lands in [`MemoryError::Other`]. The typed variants — -/// `Invalid`, `NotFound`, `Unsupported` — are constructed *here*, by the -/// driver, where the reason is actually known. -pub(super) fn engine_error(error: anyhow::Error) -> MemoryError { - MemoryError::Other(error) -} - -/// Maps a host-layer `Result<_, String>` failure onto the contract's error -/// type, tagging it with the contract method that produced it. -/// -/// The host's memory layers are `String`-typed end to end, so nothing about the -/// failure is machine-readable and [`MemoryError::Other`] is the honest -/// variant. A family that can genuinely identify a caller error — see -/// `tool_memory`'s `classify_put_rule` — constructs [`MemoryError::Invalid`] -/// itself rather than widening this helper. -pub(super) fn host_error(context: &str, error: String) -> MemoryError { - log::warn!("[memory:driver:embedded] {context} failed: {error}"); - MemoryError::Other(anyhow::anyhow!("{context}: {error}")) -} - -#[async_trait] -impl MemoryProvider for EmbeddedMemoryProvider { - fn driver_id(&self) -> &str { - EMBEDDED_DRIVER_ID - } - - fn capabilities(&self) -> Capabilities { - advertised_capabilities() - } - - async fn health(&self) -> MemoryHealth { - // Deliberately does **not** force the client. `health` is called on - // bind and for status output; making it the thing that opens SQLite - // would move the I/O the lazy `OnceCell` exists to defer back onto the - // status path — and, worse, would create the workspace as a side - // effect of asking whether it exists. - let Some(client) = self.client.get() else { - return MemoryHealth::Ready; - }; - if client.memory_handle().health_check().await { - MemoryHealth::Ready - } else { - // No path in the reason: this string is logged and rendered in - // `subsystems_status`. - MemoryHealth::down("memory workspace or database file is missing") - } - } - - // The optional-family accessors. Each must move in lockstep with - // `advertised_capabilities`; `audit_provider` fails on either half alone. - fn as_documents(&self) -> Option<&dyn tinycortex_api::provider::MemoryDocuments> { - Some(self) - } - - fn as_graph(&self) -> Option<&dyn tinycortex_api::provider::MemoryGraph> { - Some(self) - } - - fn as_tool_memory(&self) -> Option<&dyn tinycortex_api::provider::MemoryToolMemory> { - Some(self) - } - - fn as_ingest(&self) -> Option<&dyn tinycortex_api::provider::MemoryIngest> { - Some(self) - } - - fn as_tree(&self) -> Option<&dyn tinycortex_api::provider::MemoryTree> { - Some(self) - } - - fn as_entities(&self) -> Option<&dyn tinycortex_api::provider::MemoryEntities> { - Some(self) - } - - /// `None` without `memory-git`, in lockstep with - /// [`advertised_capabilities`] — `audit_provider` fails on either half - /// alone, which is exactly the check that keeps these two from drifting. - fn as_diff(&self) -> Option<&dyn tinycortex_api::provider::MemoryDiff> { - #[cfg(not(feature = "memory-git"))] - { - None - } - #[cfg(feature = "memory-git")] - { - Some(self) - } - } - - fn as_goals(&self) -> Option<&dyn tinycortex_api::provider::MemoryGoals> { - Some(self) - } - - fn as_sources(&self) -> Option<&dyn tinycortex_api::provider::MemorySourceSink> { - Some(self) - } - - fn as_maintenance(&self) -> Option<&dyn tinycortex_api::provider::MemoryMaintenance> { - Some(self) - } - - // `shutdown` keeps the contract's no-op default on purpose. The ingestion - // worker belongs to the shared `MemoryClient`, which other subsystems hold - // too; a driver must not tear down a handle it does not own. The default is - // idempotent by construction, which the contract requires. -} - -#[cfg(test)] -#[path = "mod_tests.rs"] -mod tests; - -#[cfg(test)] -pub(super) mod test_support { - use super::*; - use tempfile::TempDir; - - /// A driver over a fresh temp workspace. The `TempDir` must outlive the - /// provider, so it is returned alongside. - /// - /// The [`Config`] `OnceCell` is **seeded** rather than left to resolve - /// through [`crate::openhuman::config::load_config_with_timeout`]: that - /// path reads the real config file and the process-global - /// `OPENHUMAN_WORKSPACE`, which would make every M3c test non-hermetic and - /// order-dependent. The seeded shape is the same one - /// `tree::retrieval::source_scope_tests::test_config` uses — a default - /// `Config` re-rooted at the temp workspace, with an inert embedder. - pub fn fresh_driver() -> (TempDir, EmbeddedMemoryProvider) { - let tmp = TempDir::new().expect("temp workspace"); - let workspace = tmp.path().join("ws"); - let provider = EmbeddedMemoryProvider::new(workspace.clone(), MemoryHooksConfig::default()); - - let mut config = Config::default(); - config.workspace_dir = workspace; - config.config_path = tmp.path().join("config.toml"); - config.memory_tree.embedding_endpoint = None; - config.memory_tree.embedding_model = None; - config.memory_tree.embedding_strict = false; - provider - .config - .set(config) - .map_err(|_| "config cell already seeded") - .expect("seed test config"); - - (tmp, provider) - } -} diff --git a/src/openhuman/memory/driver/embedded/mod_tests.rs b/src/openhuman/memory/driver/embedded/mod_tests.rs deleted file mode 100644 index 645a321267..0000000000 --- a/src/openhuman/memory/driver/embedded/mod_tests.rs +++ /dev/null @@ -1,224 +0,0 @@ -//! Identity + construction tests for the embedded driver. -//! -//! The load-bearing one is -//! `constructing_the_driver_does_no_io_and_needs_no_runtime`: it is a plain -//! `#[test]` on purpose. `MemoryClient::from_workspace_dir` spawns the -//! ingestion worker, so eagerly resolving the client would panic here — and -//! would panic identically in the ~4000 pre-boot unit tests that reach -//! `CoreContext::memory_binding` with no runtime. - -use super::test_support::fresh_driver; -use super::*; - -use tinycortex_api::capabilities::Capability; -use tinycortex_api::null::NULL_DRIVER_ID; -use tinycortex_api::provider::{audit_provider, MemoryCore}; - -#[test] -fn embedded_driver_id_is_tinycortex() { - let (_tmp, provider) = fresh_driver(); - assert_eq!(provider.driver_id(), EMBEDDED_DRIVER_ID); - assert_ne!(provider.driver_id(), NULL_DRIVER_ID); -} - -#[test] -fn embedded_driver_advertises_the_mandatory_three() { - let (_tmp, provider) = fresh_driver(); - let advertised = provider.capabilities(); - - for mandatory in Capability::MANDATORY { - assert!( - advertised.contains(mandatory), - "{mandatory} must be advertised" - ); - } -} - -#[test] -fn embedded_driver_advertises_documents_graph_and_tool_memory() { - let (_tmp, provider) = fresh_driver(); - let advertised = provider.capabilities(); - - for landed in [ - Capability::Documents, - Capability::Graph, - Capability::ToolMemory, - ] { - assert!(advertised.contains(landed), "{landed} landed in M3b"); - } -} - -#[test] -fn embedded_driver_advertises_ingest_tree_and_entities() { - let (_tmp, provider) = fresh_driver(); - let advertised = provider.capabilities(); - - for landed in [Capability::Ingest, Capability::Tree, Capability::Entities] { - assert!(advertised.contains(landed), "{landed} landed in M3c"); - } -} - -#[test] -fn embedded_driver_advertises_diff_goals_sources_and_maintenance() { - let (_tmp, provider) = fresh_driver(); - let advertised = provider.capabilities(); - - for landed in [ - Capability::Diff, - Capability::Goals, - Capability::Sources, - Capability::Maintenance, - ] { - assert!(advertised.contains(landed), "{landed} landed in M3d"); - } -} - -/// The assertion the whole M3 milestone was aimed at. -#[test] -fn embedded_driver_advertises_every_capability() { - let (_tmp, provider) = fresh_driver(); - assert_eq!( - provider.capabilities(), - Capabilities::all(), - "a bound context must advertise the same thirteen families an unbound one does" - ); - // Equality alone would still hold with every `as_*` returning `None`; that - // is what `embedded_driver_passes_capability_audit` is for. - for family in Capability::ALL { - assert!(provider.capabilities().contains(family), "{family} missing"); - } -} - -#[test] -fn bound_and_unbound_contexts_agree_on_the_capability_set() { - use crate::openhuman::memory::binding; - - // The inversion this milestone existed to fix: before M3, binding a driver - // *narrowed* the advertised set from thirteen families to three, so a bound - // host looked less capable than an unbound one. - let dir = tempfile::TempDir::new().unwrap(); - let binding = binding::for_workspace( - dir.path(), - &crate::openhuman::config::schema::MemorySubsystemConfig::default(), - ) - .expect("default bind"); - - assert_eq!( - binding.capabilities(), - binding::unbound_default_capabilities() - ); -} - -#[test] -fn embedded_driver_passes_capability_audit() { - let (_tmp, provider) = fresh_driver(); - assert!( - audit_provider(&provider).is_ok(), - "advertised set and reachable accessors disagree: {:?}", - audit_provider(&provider).err() - ); -} - -#[test] -fn constructing_the_driver_does_no_io_and_needs_no_runtime() { - // Deliberately NOT `#[tokio::test]`, and deliberately a path that does not - // exist: construction must neither spawn nor create anything. - let tmp = tempfile::TempDir::new().unwrap(); - let workspace = tmp.path().join("never-created"); - - let provider = EmbeddedMemoryProvider::new(workspace.clone(), MemoryHooksConfig::default()); - - assert_eq!(provider.workspace_dir(), workspace.as_path()); - assert!( - !workspace.exists(), - "constructing the driver must not create the workspace" - ); -} - -#[tokio::test] -async fn health_does_not_force_client_construction() { - let tmp = tempfile::TempDir::new().unwrap(); - let workspace = tmp.path().join("never-created"); - let provider = EmbeddedMemoryProvider::new(workspace.clone(), MemoryHooksConfig::default()); - - assert_eq!(provider.health().await, MemoryHealth::Ready); - assert!( - !workspace.exists(), - "a health probe must not open (or create) the store" - ); -} - -#[tokio::test] -async fn health_is_ready_once_the_client_is_resolved() { - let (_tmp, provider) = fresh_driver(); - // Force resolution through a real contract call. - provider.namespaces().await.expect("namespaces"); - - assert_eq!(provider.health().await, MemoryHealth::Ready); -} - -#[tokio::test] -async fn shutdown_is_a_no_op_and_is_idempotent() { - let (_tmp, provider) = fresh_driver(); - provider.shutdown().await.expect("first shutdown"); - provider.shutdown().await.expect("second shutdown"); -} - -/// What the trait indirection actually costs, measured rather than asserted. -/// -/// `docs/specs/plan-memory.md` §9 flags `#[async_trait]` dispatch as a -/// performance risk for the recall path. An end-to-end recall p50 cannot answer -/// that question — it is dominated by SQLite and, on the semantic path, by an -/// embedding call, both of which swamp a vtable hop by several orders of -/// magnitude. So this measures the *same* call twice, once statically and once -/// through `Arc`; the storage cost is identical in both arms -/// and only dispatch differs, so the delta is the indirection. -/// -/// `#[ignore]`d and assertion-free on purpose: it prints, it does not gate. -/// -/// ```text -/// GGML_NATIVE=OFF cargo test --lib \ -/// openhuman::memory::driver::embedded::tests::trait_indirection_dispatch_cost \ -/// -- --ignored --nocapture -/// ``` -#[tokio::test] -#[ignore = "microbenchmark: prints ns/op, asserts nothing"] -async fn trait_indirection_dispatch_cost() { - const ITERATIONS: u32 = 20_000; - - let (_tmp, provider) = fresh_driver(); - // Warm the lazy client so its one-time construction is not counted. - provider.get("bench_ns", "absent").await.expect("warmup"); - - let started = std::time::Instant::now(); - for _ in 0..ITERATIONS { - let _ = provider.get("bench_ns", "absent").await; - } - let statik = started.elapsed(); - - let dynamic_provider: Arc = Arc::new(provider); - let started = std::time::Instant::now(); - for _ in 0..ITERATIONS { - let _ = dynamic_provider.get("bench_ns", "absent").await; - } - let dynamic = started.elapsed(); - - let per_call = |d: std::time::Duration| d.as_nanos() as f64 / f64::from(ITERATIONS); - println!( - "[bench] MemoryCore::get static={:.0}ns/op dyn={:.0}ns/op delta={:+.0}ns/op", - per_call(statik), - per_call(dynamic), - per_call(dynamic) - per_call(statik) - ); -} - -#[test] -fn hooks_are_carried_through_from_config() { - let tmp = tempfile::TempDir::new().unwrap(); - let hooks = MemoryHooksConfig { - auto_recall: false, - ..MemoryHooksConfig::default() - }; - let provider = EmbeddedMemoryProvider::new(tmp.path().join("ws"), hooks); - assert!(!provider.hooks().auto_recall); -} diff --git a/src/openhuman/memory/driver/embedded/portability.rs b/src/openhuman/memory/driver/embedded/portability.rs deleted file mode 100644 index fdba6e001a..0000000000 --- a/src/openhuman/memory/driver/embedded/portability.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! [`MemoryPortability`] for the embedded driver — the export/import pair that -//! makes binding this driver reversible. -//! -//! ## There was no host entry point, so this one is composed -//! -//! Unlike the other mandatory families, portability had nothing to delegate to. -//! It is composed from two existing engine calls — `namespace_summaries()` and -//! `list()` — and deliberately *not* from `MemoryClient::list_documents`, whose -//! SQL selects `document_id, namespace, key, title, source_type, priority, -//! created_at, updated_at, taint` and **no `content`**: an export built on it -//! would round-trip metadata and lose every byte of memory. -//! -//! ## Fidelity, stated plainly -//! -//! A record round-trips the five fields [`MemoryCore`](tinycortex_api::provider::MemoryCore) -//! owns — `key`, `content`, `category`, `session_id`, `taint` — plus its -//! namespace and export-time timestamp. Document-tier attributes (`title`, -//! `tags`, `metadata`, `source_type`, `priority`) belong to the -//! [`Documents`](tinycortex_api::capabilities::Capability::Documents) family and -//! are out of scope here; a re-import synthesises them the same way a normal -//! store does (`title = key`, `source_type = "chat"`). That is a real -//! limitation of the mandatory-only export, not an oversight — it widens when -//! the Documents family lands. -//! -//! ## Cursor -//! -//! `"{namespace_index}:{offset}"`, indexing into `namespace_summaries()`, whose -//! SQL is `ORDER BY namespace` and therefore stable. `None` starts at `"0:0"`. -//! A cursor that does not parse, or whose index is not a namespace this driver -//! holds, is [`MemoryError::Invalid`] — the contract names exactly that case. -//! Note an empty page is **not** a terminator: only a `None` next-cursor is, so -//! an empty namespace advances the index and keeps paging. - -use async_trait::async_trait; -use serde_json::json; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::{ExportPage, ExportRecord, ImportOutcome}; -use tinycortex_api::provider::MemoryPortability; -use tinycortex_api::types::{MemoryCategory, MemoryEntry, GLOBAL_NAMESPACE}; - -use super::{engine_error, EmbeddedMemoryProvider}; - -/// The [`ExportRecord::kind`] this driver emits and accepts. -pub(super) const ENTRY_KIND: &str = "entry"; - -/// Parses `"{index}:{offset}"`. `None` means "start". -fn parse_cursor(cursor: Option<&str>) -> Result<(usize, usize), MemoryError> { - let Some(raw) = cursor else { - return Ok((0, 0)); - }; - let invalid = - || MemoryError::Invalid(format!("export cursor not issued by this driver: {raw}")); - let (index, offset) = raw.split_once(':').ok_or_else(invalid)?; - Ok(( - index.parse().map_err(|_| invalid())?, - offset.parse().map_err(|_| invalid())?, - )) -} - -fn to_record(entry: MemoryEntry) -> ExportRecord { - ExportRecord { - kind: ENTRY_KIND.to_string(), - id: entry.id, - namespace: entry.namespace, - taint: entry.taint, - payload: json!({ - "key": entry.key, - "content": entry.content, - "category": entry.category.to_string(), - "session_id": entry.session_id, - "timestamp": entry.timestamp, - }), - } -} - -/// What `import_records` needs out of one record's payload. -struct ImportedEntry { - namespace: String, - key: String, - content: String, - category: MemoryCategory, - session_id: Option, -} - -/// Reads a record into the fields the engine's store needs. -/// -/// # Errors -/// -/// An operator-facing reason with **no record content in it** — these strings -/// land in [`ImportOutcome::errors`], which is logged. -fn read_record(record: &ExportRecord) -> Result { - if record.kind != ENTRY_KIND { - return Err(format!( - "record {}: unsupported kind '{}' (this driver exports '{ENTRY_KIND}')", - record.id, record.kind - )); - } - let key = record - .payload - .get("key") - .and_then(|v| v.as_str()) - .ok_or_else(|| format!("record {}: payload is missing a string 'key'", record.id))?; - let content = record - .payload - .get("content") - .and_then(|v| v.as_str()) - .ok_or_else(|| { - format!( - "record {}: payload is missing a string 'content'", - record.id - ) - })?; - let raw_category = record - .payload - .get("category") - .and_then(|v| v.as_str()) - .ok_or_else(|| { - format!( - "record {}: payload is missing a string 'category'", - record.id - ) - })?; - let category: MemoryCategory = raw_category - .parse() - .map_err(|_| format!("record {}: category is not a known category", record.id))?; - - Ok(ImportedEntry { - namespace: record - .namespace - .clone() - .unwrap_or_else(|| GLOBAL_NAMESPACE.to_string()), - key: key.to_string(), - content: content.to_string(), - category, - session_id: record - .payload - .get("session_id") - .and_then(|v| v.as_str()) - .map(str::to_string), - }) -} - -#[async_trait] -impl MemoryPortability for EmbeddedMemoryProvider { - async fn export_page( - &self, - cursor: Option<&str>, - limit: usize, - ) -> Result { - if limit == 0 { - return Err(MemoryError::Invalid( - "export page limit must be greater than zero".to_string(), - )); - } - let (index, offset) = parse_cursor(cursor)?; - let memory = self.memory().await?; - let summaries = memory.namespace_summaries().await.map_err(engine_error)?; - - if index >= summaries.len() { - // A start-of-export against an empty store lands here legitimately; - // any other out-of-range index came from a cursor we did not issue. - if cursor.is_some() && !summaries.is_empty() { - return Err(MemoryError::Invalid(format!( - "export cursor names namespace #{index}, but this driver holds {}", - summaries.len() - ))); - } - return Ok(ExportPage { - records: Vec::new(), - next_cursor: None, - }); - } - - let namespace = &summaries[index].namespace; - let entries = memory - .list(Some(namespace), None, None) - .await - .map_err(engine_error)?; - if offset > entries.len() { - return Err(MemoryError::Invalid(format!( - "export cursor offset {offset} is past the end of namespace #{index}" - ))); - } - - let end = offset.saturating_add(limit).min(entries.len()); - let records: Vec = entries[offset..end] - .iter() - .cloned() - .map(to_record) - .collect(); - - let next_cursor = if end < entries.len() { - Some(format!("{index}:{end}")) - } else if index + 1 < summaries.len() { - Some(format!("{}:0", index + 1)) - } else { - None - }; - - log::debug!( - "[memory:driver:embedded] export_page index={index} offset={offset} \ - emitted={} more={}", - records.len(), - next_cursor.is_some() - ); - Ok(ExportPage { - records, - next_cursor, - }) - } - - async fn import_records( - &self, - records: Vec, - ) -> Result { - let memory = self.memory().await?; - let mut outcome = ImportOutcome::default(); - - for record in records { - let entry = match read_record(&record) { - Ok(entry) => entry, - Err(reason) => { - // Per-record rejection is reported, never fatal: a - // million-record restore must not abort on one bad row. - outcome.failed = outcome.failed.saturating_add(1); - outcome.errors.push(reason); - continue; - } - }; - - // `store_with_taint` with the record's *own* taint: an importing - // driver must persist what it is given and must not re-stamp - // provenance. `Memory::store` would stamp `Internal`. - memory - .store_with_taint( - &entry.namespace, - &entry.key, - &entry.content, - entry.category, - entry.session_id.as_deref(), - record.taint, - ) - .await - .map_err(engine_error)?; - outcome.imported = outcome.imported.saturating_add(1); - } - - log::debug!( - "[memory:driver:embedded] import_records imported={} failed={}", - outcome.imported, - outcome.failed - ); - Ok(outcome) - } -} - -#[cfg(test)] -#[path = "portability_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/portability_tests.rs b/src/openhuman/memory/driver/embedded/portability_tests.rs deleted file mode 100644 index 1ad856c004..0000000000 --- a/src/openhuman/memory/driver/embedded/portability_tests.rs +++ /dev/null @@ -1,237 +0,0 @@ -//! [`MemoryPortability`] tests. -//! -//! `export_import_round_trips_content_category_session_and_taint` is the one -//! that makes the binding reversible in practice, and -//! `import_does_not_restamp_provenance` is its security half — an import that -//! stamped `Internal` would silently upgrade externally-sourced content on -//! every migration. - -use super::super::test_support::fresh_driver; -use super::*; - -use tempfile::TempDir; -use tinycortex_api::provider::MemoryCore; -use tinycortex_api::types::MemoryTaint; - -use crate::openhuman::memory::driver::embedded::EmbeddedMemoryProvider; - -/// Drains the whole export, asserting the loop terminates on a `None` cursor. -async fn export_all(provider: &EmbeddedMemoryProvider, limit: usize) -> Vec { - let mut cursor: Option = None; - let mut out = Vec::new(); - for _ in 0..64 { - let page = provider - .export_page(cursor.as_deref(), limit) - .await - .expect("export page"); - out.extend(page.records); - match page.next_cursor { - Some(next) => cursor = Some(next), - None => return out, - } - } - panic!("export did not terminate within 64 pages"); -} - -async fn seed(provider: &EmbeddedMemoryProvider) { - provider - .store( - "ns_a", - "a1", - "first in a", - MemoryCategory::Core, - Some("sess-1"), - MemoryTaint::Internal, - ) - .await - .expect("store a1"); - provider - .store( - "ns_a", - "a2", - "second in a", - MemoryCategory::Custom("notes".into()), - None, - MemoryTaint::ExternalSync, - ) - .await - .expect("store a2"); - provider - .store( - "ns_b", - "b1", - "first in b", - MemoryCategory::Daily, - None, - MemoryTaint::Internal, - ) - .await - .expect("store b1"); -} - -#[tokio::test] -async fn export_of_an_empty_store_terminates_immediately() { - let (_tmp, provider) = fresh_driver(); - let page = provider.export_page(None, 10).await.expect("export"); - assert!(page.records.is_empty()); - assert!(page.next_cursor.is_none()); -} - -#[tokio::test] -async fn export_paginates_across_namespaces_and_terminates_on_a_null_cursor() { - let (_tmp, provider) = fresh_driver(); - seed(&provider).await; - - // A page size of 1 forces both intra-namespace and cross-namespace cursor - // advances. - let records = export_all(&provider, 1).await; - assert_eq!(records.len(), 3, "every seeded entry must be exported"); - - let mut namespaces: Vec<&str> = records - .iter() - .filter_map(|r| r.namespace.as_deref()) - .collect(); - namespaces.sort_unstable(); - namespaces.dedup(); - assert_eq!(namespaces, vec!["ns_a", "ns_b"]); - assert!(records.iter().all(|r| r.kind == ENTRY_KIND)); -} - -#[tokio::test] -async fn export_import_round_trips_content_category_session_and_taint() { - let (_source_tmp, source) = fresh_driver(); - seed(&source).await; - let records = export_all(&source, 2).await; - - // Import into a *second*, independent workspace. - let target_tmp = TempDir::new().unwrap(); - let target = EmbeddedMemoryProvider::new( - target_tmp.path().join("ws"), - crate::openhuman::config::schema::MemoryHooksConfig::default(), - ); - let outcome = target.import_records(records).await.expect("import"); - assert_eq!(outcome.imported, 3); - assert_eq!(outcome.failed, 0); - assert!(outcome.errors.is_empty()); - - let a1 = target - .get("ns_a", "a1") - .await - .expect("get") - .expect("a1 imported"); - assert_eq!(a1.content, "first in a"); - assert_eq!(a1.category, MemoryCategory::Core); - assert_eq!(a1.taint, MemoryTaint::Internal); - - let a2 = target - .get("ns_a", "a2") - .await - .expect("get") - .expect("a2 imported"); - assert_eq!(a2.content, "second in a"); - assert_eq!(a2.category, MemoryCategory::Custom("notes".into())); - assert_eq!(a2.taint, MemoryTaint::ExternalSync); - - let b1 = target - .get("ns_b", "b1") - .await - .expect("get") - .expect("b1 imported"); - assert_eq!(b1.category, MemoryCategory::Daily); - - // And the export of the target reproduces the same set. - let reexported = export_all(&target, 10).await; - assert_eq!(reexported.len(), 3); -} - -/// SECURITY: an importing driver persists the taint it is given. -#[tokio::test] -async fn import_does_not_restamp_provenance() { - let (_tmp, provider) = fresh_driver(); - let record = ExportRecord { - kind: ENTRY_KIND.to_string(), - id: "doc-1".into(), - namespace: Some("ns_a".into()), - taint: MemoryTaint::ExternalSync, - payload: json!({ - "key": "synced", - "content": "from elsewhere", - "category": "core", - "session_id": serde_json::Value::Null, - "timestamp": "2026-01-01T00:00:00Z", - }), - }; - - let outcome = provider.import_records(vec![record]).await.expect("import"); - assert_eq!(outcome.imported, 1); - - let got = provider - .get("ns_a", "synced") - .await - .expect("get") - .expect("imported"); - assert_eq!(got.taint, MemoryTaint::ExternalSync); -} - -#[tokio::test] -async fn import_reports_bad_records_as_failed_with_a_reason_and_does_not_abort_the_batch() { - let (_tmp, provider) = fresh_driver(); - let good = ExportRecord { - kind: ENTRY_KIND.to_string(), - id: "doc-good".into(), - namespace: Some("ns_a".into()), - taint: MemoryTaint::Internal, - payload: json!({ - "key": "kept", - "content": "SECRET-CONTENT-MARKER", - "category": "core", - }), - }; - let unknown_kind = ExportRecord { - kind: "chunk".into(), - id: "doc-chunk".into(), - namespace: Some("ns_a".into()), - taint: MemoryTaint::Internal, - payload: json!({ "content": "SECRET-CONTENT-MARKER" }), - }; - let malformed = ExportRecord { - kind: ENTRY_KIND.to_string(), - id: "doc-bad".into(), - namespace: Some("ns_a".into()), - taint: MemoryTaint::Internal, - payload: json!({ "content": "SECRET-CONTENT-MARKER" }), - }; - - let outcome = provider - .import_records(vec![unknown_kind, good, malformed]) - .await - .expect("a malformed record must not fail the batch"); - - assert_eq!(outcome.imported, 1); - assert_eq!(outcome.failed, 2); - assert_eq!(outcome.errors.len(), 2); - for error in &outcome.errors { - assert!( - !error.contains("SECRET-CONTENT-MARKER"), - "import errors are logged and must not carry record content: {error}" - ); - } - assert!(provider.get("ns_a", "kept").await.expect("get").is_some()); -} - -#[tokio::test] -async fn export_rejects_a_cursor_this_driver_did_not_issue() { - let (_tmp, provider) = fresh_driver(); - seed(&provider).await; - - for bad in ["nonsense", "1", "a:b", "99:0", "0:9999"] { - match provider.export_page(Some(bad), 10).await { - Err(MemoryError::Invalid(_)) => {} - Err(other) => panic!("cursor '{bad}' produced the wrong variant: {other:?}"), - Ok(page) => panic!( - "cursor '{bad}' must be rejected, got {} record(s)", - page.records.len() - ), - } - } -} diff --git a/src/openhuman/memory/driver/embedded/recall.rs b/src/openhuman/memory/driver/embedded/recall.rs deleted file mode 100644 index 7f89fe1a38..0000000000 --- a/src/openhuman/memory/driver/embedded/recall.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! [`MemoryRecall`] for the embedded driver. -//! -//! The delegation itself is trivial — [`OwnedRecallOpts`] borrows into the -//! engine's `RecallOpts` for free, and `UnifiedMemory::recall` already returns -//! ranked, `min_score`-filtered, category-filtered, cross-session-merged -//! results. Nothing is re-ranked here. -//! -//! ## `scope` is refused, not ignored -//! -//! The contract's `scope` is a **query predicate the driver must apply -//! internally**: applying it after the fact would let `limit` be consumed by -//! rows the caller may not see, and an empty scope denies all source-attributed -//! content rather than waving it through. -//! -//! The embedded recall path has no such predicate today. `Memory::recall` → -//! `query_namespace_ranked_excluding_session` consults nothing resembling -//! [`SourceScope`]; the host's ambient equivalent -//! ([`crate::openhuman::memory::source_scope`]) is read only by the -//! tree-retrieval and chunk-search layers, which land with the -//! [`Tree`](tinycortex_api::capabilities::Capability::Tree) family. -//! -//! So a `Some(scope)` here has exactly three possible treatments, and two are -//! wrong: -//! -//! - *silently ignore it* — a scoped query answered in full. That is a leak, -//! and it is invisible. -//! - *post-filter the rows* — the failure mode the contract's own docs name. -//! - *refuse* — what this driver does, until the predicate exists. -//! -//! [`MemoryError::Invalid`] is the right variant, not `Unsupported`: -//! `Unsupported` names a whole capability *family*, and recall is advertised. -//! Nothing calls the driver's `recall` yet (the RPC surface still goes straight -//! to the engine), so the refusal costs no live behaviour — it just cannot be -//! mistaken for working. - -use async_trait::async_trait; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::SourceScope; -use tinycortex_api::provider::MemoryRecall; -use tinycortex_api::recall::OwnedRecallOpts; -use tinycortex_api::types::{MemoryEntry, RecallOpts}; - -use super::{engine_error, EmbeddedMemoryProvider}; - -/// Refusal message for a scoped recall. A constant so the test asserts the same -/// string the caller sees. -pub(super) const SCOPE_UNAPPLIED: &str = - "source scope is not applied by the embedded recall path yet: the scope predicate lives in \ - the tree-retrieval layer and lands with the tree capability family"; - -#[async_trait] -impl MemoryRecall for EmbeddedMemoryProvider { - async fn recall( - &self, - query: &str, - limit: usize, - opts: &OwnedRecallOpts, - scope: Option<&SourceScope>, - ) -> Result, MemoryError> { - if scope.is_some() { - log::warn!("[memory:driver:embedded] recall refused: {SCOPE_UNAPPLIED}"); - return Err(MemoryError::Invalid(SCOPE_UNAPPLIED.to_string())); - } - - log::debug!( - "[memory:driver:embedded] recall query_len={} limit={limit} namespace={} \ - cross_session={}", - query.len(), - opts.namespace.as_deref().unwrap_or("-"), - opts.cross_session - ); - - // Zero-copy for the string filters; exhaustively destructured inside - // the contract crate so a new filter cannot be dropped silently. - let borrowed = RecallOpts::from(opts); - self.memory() - .await? - .recall(query, limit, borrowed) - .await - .map_err(engine_error) - } -} - -#[cfg(test)] -#[path = "recall_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/recall_tests.rs b/src/openhuman/memory/driver/embedded/recall_tests.rs deleted file mode 100644 index f54cab2fb0..0000000000 --- a/src/openhuman/memory/driver/embedded/recall_tests.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! [`MemoryRecall`] tests. -//! -//! `recall_with_a_source_scope_is_refused_until_the_predicate_exists` is the -//! one that matters: it pins the deliberate refusal so nobody "fixes" it by -//! quietly dropping the argument, which would answer a scoped query in full. - -use super::super::test_support::fresh_driver; -use super::*; - -use tinycortex_api::provider::types::SourceScope; -use tinycortex_api::provider::MemoryCore; -use tinycortex_api::types::{MemoryCategory, MemoryTaint}; - -async fn seed(provider: &EmbeddedMemoryProvider) { - provider - .store( - "ns_a", - "rust", - "the rust programming language is memory safe", - MemoryCategory::Core, - None, - MemoryTaint::Internal, - ) - .await - .expect("store rust"); - provider - .store( - "ns_a", - "sailing", - "sailing boats need wind", - MemoryCategory::Core, - None, - MemoryTaint::Internal, - ) - .await - .expect("store sailing"); -} - -fn opts_for(namespace: &str) -> OwnedRecallOpts { - OwnedRecallOpts { - namespace: Some(namespace.to_string()), - ..OwnedRecallOpts::default() - } -} - -#[tokio::test] -async fn recall_returns_ranked_results_for_a_query() { - let (_tmp, provider) = fresh_driver(); - seed(&provider).await; - - let hits = provider - .recall("rust programming language", 5, &opts_for("ns_a"), None) - .await - .expect("recall"); - - assert!(!hits.is_empty(), "expected at least one hit"); - let scores: Vec = hits.iter().map(|h| h.score.unwrap_or(0.0)).collect(); - assert!( - scores.windows(2).all(|w| w[0] >= w[1]), - "results must be most-relevant first: {scores:?}" - ); - assert_eq!(hits[0].key, "rust"); -} - -#[tokio::test] -async fn recall_of_an_empty_store_is_empty_not_an_error() { - let (_tmp, provider) = fresh_driver(); - let hits = provider - .recall("anything", 5, &opts_for("ns_a"), None) - .await - .expect("recall must not error on an empty store"); - assert!(hits.is_empty()); -} - -#[tokio::test] -async fn recall_honours_the_min_score_filter_from_owned_opts() { - let (_tmp, provider) = fresh_driver(); - seed(&provider).await; - - let opts = OwnedRecallOpts { - min_score: Some(1.1), - ..opts_for("ns_a") - }; - let hits = provider - .recall("rust programming language", 5, &opts, None) - .await - .expect("recall"); - assert!( - hits.is_empty(), - "an unreachable min_score must drop every hit, got {}", - hits.len() - ); -} - -#[tokio::test] -async fn recall_surfaces_external_sync_taint() { - let (_tmp, provider) = fresh_driver(); - provider - .store( - "ns_a", - "synced", - "kubernetes cluster autoscaling notes", - MemoryCategory::Core, - None, - MemoryTaint::ExternalSync, - ) - .await - .expect("store"); - - let hits = provider - .recall("kubernetes cluster autoscaling", 5, &opts_for("ns_a"), None) - .await - .expect("recall"); - let hit = hits.first().expect("expected a hit"); - assert_eq!(hit.taint, MemoryTaint::ExternalSync); -} - -#[tokio::test] -async fn recall_with_a_source_scope_is_refused_until_the_predicate_exists() { - let (_tmp, provider) = fresh_driver(); - seed(&provider).await; - - // The unscoped call still works … - provider - .recall("rust", 5, &opts_for("ns_a"), None) - .await - .expect("unscoped recall"); - - // … while a scoped one is refused loudly rather than answered in full. - let scope = SourceScope::new(["src-abc"]); - let error = provider - .recall("rust", 5, &opts_for("ns_a"), Some(&scope)) - .await - .expect_err("a scope this driver cannot apply must be refused"); - match error { - MemoryError::Invalid(reason) => assert_eq!(reason, SCOPE_UNAPPLIED), - other => panic!("expected MemoryError::Invalid, got {other:?}"), - } - - // An *empty* scope denies all source-attributed content, so it must be - // refused too rather than read as "unrestricted". - let empty = SourceScope::default(); - assert!(provider - .recall("rust", 5, &opts_for("ns_a"), Some(&empty)) - .await - .is_err()); -} diff --git a/src/openhuman/memory/driver/embedded/sources.rs b/src/openhuman/memory/driver/embedded/sources.rs deleted file mode 100644 index 1d364625ed..0000000000 --- a/src/openhuman/memory/driver/embedded/sources.rs +++ /dev/null @@ -1,214 +0,0 @@ -//! [`MemorySourceSink`] for the embedded driver — the write seam the host's -//! sync machinery pushes already-fetched items through. -//! -//! ## The host keeps the loop; this file is one step of it -//! -//! `memory::sources::sync::sync_source` stays exactly where it is. It owns the -//! per-source mutex, the `emit_sync_stage` progress events, Composio billing, -//! OAuth, rate limits, and dispatch by `SourceKind` — none of which belongs -//! behind a trait a third-party driver implements. This family is only the -//! "persist these items" step at the end of that loop. -//! -//! ## TAINT — why this family writes documents and `MemoryIngest` refuses -//! -//! `taint` is host-stamped provenance a driver "must persist … and never assign -//! itself". The two write tiers differ on whether they can: -//! -//! - The **chunk** tier (`ingest_pipeline::ingest_document_with_scope`, what -//! `run_source_pipeline` writes through) has **no taint column** anywhere -//! along its path — not on `DocumentInput`, not on `CanonicalisedSource`, not -//! on `Chunk::metadata`. `MemoryIngest` (M3c) therefore *refuses* a non-default -//! taint rather than dropping it. -//! - The **namespace-document** tier (`MemoryClient::put_doc` → -//! `NamespaceDocumentInput.taint`) carries it as a real column. -//! -//! Refusal is the right answer for `MemoryIngest`, whose callers pass the -//! default. It is the *wrong* answer here: the contract says sync paths pass -//! [`MemoryTaint::ExternalSync`], so a sink that refuses non-default taint would -//! refuse its only intended caller. So this family writes through the tier that -//! can honour the argument. The alternative — call the chunk path and let the -//! `taint` parameter fall on the floor — is the exact failure the rule exists to -//! prevent: externally-synced content landing indistinguishable from -//! user-authored content across a prompt-injection trust boundary. -//! -//! **The behaviour difference this buys is real and is not hidden.** Items -//! accepted here land as namespace documents, so they are readable through -//! `MemoryDocuments` / `MemoryCore` and are queryable, but they do **not** flow -//! through canonicalisation, chunking and the summary-tree ingest the way -//! `run_source_pipeline` output does — so they do not appear in a source's -//! summary tree. Closing that gap needs a taint column on the chunk tier, not a -//! different call here. -//! -//! ## `skipped` is always 0, deliberately -//! -//! `put_doc` upserts and returns a document id whether the key was new or -//! already present; there is no already-present signal to read. Reporting a -//! guess would corrupt the one number the contract gives a caller for detecting -//! a silently-dropping driver, so the honest value is 0 and the count lands in -//! `written`. -//! -//! ## Naming hazard -//! -//! `tinycortex::memory::sources::SourceItem` also exists and is a **different -//! type** (an engine-side sync item). Everything here is -//! [`tinycortex_api::provider::types::SourceItem`]; nothing imports the other -//! one. - -use async_trait::async_trait; -use serde_json::json; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::{IngestOutcome, SourceItem}; -use tinycortex_api::provider::MemorySourceSink; -use tinycortex_api::types::{MemoryTaint, NamespaceDocumentInput}; - -use crate::openhuman::memory::store::chunks::store as chunks; -use crate::openhuman::memory::store::chunks::types::SourceKind; - -use super::{host_error, EmbeddedMemoryProvider}; - -/// The namespace one logical source's accepted items live in. -/// -/// Keyed on `source_id` alone — **not** on `(source_kind, source_id)` — so -/// [`MemorySourceSink::forget_source`], which is given only the id, can address -/// exactly what [`MemorySourceSink::accept_source_items`] wrote. The kind is -/// carried on each document instead (`source_type` + metadata), where losing it -/// costs nothing. -fn namespace_for(source_id: &str) -> String { - format!("source:{source_id}") -} - -#[async_trait] -impl MemorySourceSink for EmbeddedMemoryProvider { - async fn accept_source_items( - &self, - source_id: &str, - source_kind: &str, - items: Vec, - taint: MemoryTaint, - ) -> Result { - log::debug!( - "[memory:driver:embedded] accept_source_items source_id={source_id} \ - source_kind={source_kind} items={} taint={}", - items.len(), - taint.as_db_str() - ); - - let namespace = namespace_for(source_id); - let client = self.client().await?; - let mut outcome = IngestOutcome::default(); - - for item in items { - if item.item_id.trim().is_empty() { - // The item id is the upsert key. An empty one would collide - // every such item onto a single document, silently losing all - // but the last — refuse instead. - return Err(MemoryError::Invalid( - "source item_id must not be empty: it is the dedupe key".to_string(), - )); - } - - let title = if item.title.trim().is_empty() { - item.item_id.clone() - } else { - item.title.clone() - }; - - let input = NamespaceDocumentInput { - namespace: namespace.clone(), - key: item.item_id, - title, - content: item.content, - source_type: source_kind.to_string(), - priority: "medium".to_string(), - tags: item.tags, - metadata: json!({ - "sourceId": source_id, - "sourceKind": source_kind, - // This is collection identity, deliberately separate - // from `item_id`, which is only the per-item dedupe key. - "path_scope": namespace, - "url": item.url, - "mime": item.mime, - "updatedAtMs": item.updated_at_ms, - }), - category: "core".to_string(), - session_id: None, - document_id: None, - // The whole point of this family. Never substituted, never - // defaulted. - taint, - }; - - let id = client - .put_doc(input) - .await - .map_err(|error| host_error("accept_source_items", error))?; - outcome.written = outcome.written.saturating_add(1); - outcome.ids.push(id); - } - - Ok(outcome) - } - - async fn forget_source(&self, source_id: &str) -> Result { - log::debug!("[memory:driver:embedded] forget_source source_id={source_id}"); - let namespace = namespace_for(source_id); - let client = self.client().await?; - - // Count before clearing: `clear_namespace` returns `()`, and the - // contract wants the number of units removed. - let listed = client - .list_documents(Some(&namespace)) - .await - .map_err(|error| host_error("forget_source", error))?; - // `list_documents` answers `{"documents": [...]}`, not a bare array. - let documents = listed - .get("documents") - .and_then(serde_json::Value::as_array) - .map(Vec::len) - .unwrap_or(0) as u64; - - if documents > 0 { - client - .clear_namespace(&namespace) - .await - .map_err(|error| host_error("forget_source", error))?; - } - - // Also drop chunk-tier content written for the same logical source - // through `MemoryIngest` — the disconnect path must not leave half the - // driver's copy of a source behind. - // - // **Exact match, never a prefix.** `sources::status::source_id_prefix` - // expands a Composio source to `{toolkit}:%`, which would take out every - // source sharing that toolkit. That over-deletion is tolerable in the - // host's own connection-teardown path, which knows it is tearing down - // the whole connection; behind a contract method that promises to drop - // *one* logical source it would be a surprise. Teardown of a Composio - // connection stays host-side in `integrations::composio::ops:: - // memory_cleanup`, which has the toolkit and connection id this family - // deliberately never sees. - let config = self.config().await?.clone(); - let id = source_id.to_string(); - let chunks_removed = tokio::task::spawn_blocking(move || -> anyhow::Result { - let removed = chunks::delete_chunks_by_source(&config, SourceKind::Document, &id)?; - // Finish off a tree left orphaned by an earlier partial delete; - // a no-op when the chunk delete above already cascaded it. - chunks::delete_orphaned_source_tree(&config, SourceKind::Document, &id)?; - Ok(removed) - }) - .await - .map_err(|error| host_error("forget_source", format!("join: {error}")))? - .map_err(|error| host_error("forget_source", format!("{error:#}")))?; - - log::debug!( - "[memory:driver:embedded] forget_source source_id={source_id} documents={documents} \ - chunks={chunks_removed}" - ); - Ok(documents + chunks_removed as u64) - } -} - -#[cfg(test)] -#[path = "sources_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/sources_tests.rs b/src/openhuman/memory/driver/embedded/sources_tests.rs deleted file mode 100644 index ab97cf2a8d..0000000000 --- a/src/openhuman/memory/driver/embedded/sources_tests.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! [`MemorySourceSink`] tests. -//! -//! `accept_source_items_persists_the_caller_supplied_taint` is the security test -//! of this family, and the reason it writes through the document tier at all: a -//! sink that quietly downgraded `external_sync` to `internal` would erase a -//! prompt-injection trust boundary while every other assertion here still -//! passed. - -use super::super::test_support::fresh_driver; -use super::*; - -fn item(item_id: &str, title: &str, content: &str) -> SourceItem { - SourceItem { - item_id: item_id.to_string(), - title: title.to_string(), - content: content.to_string(), - mime: Some("text/plain".to_string()), - url: Some(format!("https://example.invalid/{item_id}")), - updated_at_ms: Some(1_700_000_000_000), - tags: vec!["synced".to_string()], - } -} - -#[tokio::test] -async fn accept_source_items_writes_one_document_per_item() { - let (_tmp, provider) = fresh_driver(); - - let outcome = provider - .accept_source_items( - "src_a", - "folder", - vec![ - item("i1", "First", "first body"), - item("i2", "Second", "second body"), - ], - MemoryTaint::ExternalSync, - ) - .await - .expect("accept_source_items"); - - assert_eq!(outcome.written, 2); - // See the module docs: `put_doc` gives no already-present signal, so a - // non-zero `skipped` here would be invented. - assert_eq!(outcome.skipped, 0); - assert_eq!(outcome.ids.len(), 2); -} - -#[tokio::test] -async fn accept_source_items_persists_the_caller_supplied_taint() { - use tinycortex_api::provider::MemoryDocuments; - - let (_tmp, provider) = fresh_driver(); - provider - .accept_source_items( - "src_a", - "folder", - vec![item("i1", "First", "first body")], - MemoryTaint::ExternalSync, - ) - .await - .expect("accept_source_items"); - - let stored = provider - .get_document("source:src_a", "i1") - .await - .expect("get_document") - .expect("document exists"); - assert_eq!( - stored.taint, - MemoryTaint::ExternalSync, - "the sink must persist the host-stamped provenance, never downgrade it" - ); - assert_eq!(stored.content, "first body"); -} - -#[tokio::test] -async fn accept_source_items_persists_a_source_level_path_scope() { - use tinycortex_api::provider::MemoryDocuments; - - let (_tmp, provider) = fresh_driver(); - provider - .accept_source_items( - "src_a", - "folder", - vec![ - item("first", "First", "one"), - item("second", "Second", "two"), - ], - MemoryTaint::ExternalSync, - ) - .await - .expect("accept_source_items"); - - for item_id in ["first", "second"] { - let stored = provider - .get_document("source:src_a", item_id) - .await - .expect("get_document") - .expect("document exists"); - assert_eq!( - stored - .metadata - .get("path_scope") - .and_then(serde_json::Value::as_str), - Some("source:src_a"), - "path scope must identify the collection rather than item `{item_id}`" - ); - } -} - -#[tokio::test] -async fn accept_source_items_upserts_on_the_item_id() { - use tinycortex_api::provider::MemoryDocuments; - - let (_tmp, provider) = fresh_driver(); - provider - .accept_source_items( - "src_a", - "folder", - vec![item("i1", "First", "v1")], - MemoryTaint::ExternalSync, - ) - .await - .expect("first accept"); - provider - .accept_source_items( - "src_a", - "folder", - vec![item("i1", "First", "v2")], - MemoryTaint::ExternalSync, - ) - .await - .expect("second accept"); - - let stored = provider - .get_document("source:src_a", "i1") - .await - .expect("get_document") - .expect("document exists"); - assert_eq!(stored.content, "v2", "item_id is the dedupe key"); -} - -#[tokio::test] -async fn accept_source_items_refuses_an_empty_item_id() { - let (_tmp, provider) = fresh_driver(); - let error = provider - .accept_source_items( - "src_a", - "folder", - vec![item("", "No id", "body")], - MemoryTaint::ExternalSync, - ) - .await - .expect_err("an empty dedupe key must be refused, not collapsed"); - assert!(matches!(error, MemoryError::Invalid(_)), "got: {error:?}"); -} - -#[tokio::test] -async fn accept_an_empty_batch_is_a_no_op() { - let (_tmp, provider) = fresh_driver(); - let outcome = provider - .accept_source_items("src_a", "folder", Vec::new(), MemoryTaint::ExternalSync) - .await - .expect("accept_source_items"); - assert_eq!(outcome, IngestOutcome::default()); -} - -#[tokio::test] -async fn forget_source_removes_what_the_sink_wrote_and_is_idempotent() { - use tinycortex_api::provider::MemoryDocuments; - - let (_tmp, provider) = fresh_driver(); - provider - .accept_source_items( - "src_a", - "folder", - vec![item("i1", "First", "a"), item("i2", "Second", "b")], - MemoryTaint::ExternalSync, - ) - .await - .expect("accept_source_items"); - - let removed = provider - .forget_source("src_a") - .await - .expect("forget_source"); - assert_eq!(removed, 2); - - assert!(provider - .get_document("source:src_a", "i1") - .await - .expect("get_document") - .is_none()); - - // Idempotent, per the contract. - assert_eq!( - provider - .forget_source("src_a") - .await - .expect("second forget"), - 0 - ); -} - -#[tokio::test] -async fn forget_source_on_an_unknown_source_is_zero_not_an_error() { - let (_tmp, provider) = fresh_driver(); - assert_eq!( - provider - .forget_source("never-synced") - .await - .expect("forget_source must be idempotent"), - 0 - ); -} - -#[tokio::test] -async fn forget_source_leaves_a_sibling_source_alone() { - use tinycortex_api::provider::MemoryDocuments; - - let (_tmp, provider) = fresh_driver(); - for source in ["src_a", "src_a_extra"] { - provider - .accept_source_items( - source, - "folder", - vec![item("i1", "First", "body")], - MemoryTaint::ExternalSync, - ) - .await - .expect("accept_source_items"); - } - - provider - .forget_source("src_a") - .await - .expect("forget_source"); - - // Exact match, never a prefix — `src_a_extra` shares a prefix with `src_a`. - assert!(provider - .get_document("source:src_a_extra", "i1") - .await - .expect("get_document") - .is_some()); -} diff --git a/src/openhuman/memory/driver/embedded/tool_memory.rs b/src/openhuman/memory/driver/embedded/tool_memory.rs deleted file mode 100644 index 7a424be16a..0000000000 --- a/src/openhuman/memory/driver/embedded/tool_memory.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! [`MemoryToolMemory`] for the embedded driver — per-tool learned rules. -//! -//! Backed by the engine's own [`ToolMemoryStore`], built over this driver's -//! `Arc` through the host wrapper -//! [`tool_memory_store`](crate::openhuman::memory::tool_memory::tool_memory_store). -//! The store is a single `Arc` behind a `#[derive(Clone)]` struct, so -//! constructing one per call costs an `Arc` clone and keeps the driver's lazy -//! client rule intact — no eager handle, no second `OnceCell`. -//! -//! ## Not through `memory::ops::tool_memory` -//! -//! Those are the RPC handlers, and their `open_store()` resolves the -//! **process-global** active memory client. This driver holds a -//! workspace-scoped client on purpose: routing through the global slot would -//! let a driver bound to workspace B write into workspace A, which is exactly -//! the property the workspace-keyed binding map buys. -//! -//! ## Taint -//! -//! `ToolMemoryStore::put_rule` writes through `Memory::store`, and that is -//! correct here. The taint trap this milestone warns about is the engine's -//! *default* `store` impl, which drops the taint argument; -//! `UnifiedMemory::store` is an explicit impl forwarding to -//! `store_with_taint(..., MemoryTaint::Internal)`. Tool rules are host-authored, -//! so `Internal` is the right provenance — and no method in this family takes a -//! taint argument to lose in the first place. - -use async_trait::async_trait; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::MemoryToolMemory; -use tinycortex_api::tool_memory::ToolMemoryRule; - -use super::{host_error, EmbeddedMemoryProvider}; -use crate::openhuman::memory::tool_memory::{tool_memory_store, ToolMemoryStore}; - -/// The two rejections `ToolMemoryStore::put_rule` performs before touching -/// storage. Matched by value so a genuine backend failure is never mislabelled -/// as caller error. -const PUT_RULE_REJECTIONS: [&str; 2] = ["tool_name is required", "rule body is required"]; - -/// Classifies a `put_rule` failure: a validated rejection is -/// [`MemoryError::Invalid`], everything else is a backend failure. -fn classify_put_rule(error: String) -> MemoryError { - if PUT_RULE_REJECTIONS.contains(&error.as_str()) { - return MemoryError::Invalid(error); - } - host_error("put_tool_rule", error) -} - -impl EmbeddedMemoryProvider { - async fn tool_memory(&self) -> Result { - Ok(tool_memory_store(self.memory().await?)) - } -} - -#[async_trait] -impl MemoryToolMemory for EmbeddedMemoryProvider { - async fn tool_rules(&self, tool_name: &str) -> Result, MemoryError> { - log::debug!("[memory:driver:embedded] tool_rules tool={tool_name}"); - self.tool_memory() - .await? - .list_rules(tool_name) - .await - .map_err(|error| host_error("tool_rules", error)) - } - - async fn put_tool_rule(&self, rule: ToolMemoryRule) -> Result<(), MemoryError> { - log::debug!( - "[memory:driver:embedded] put_tool_rule tool={} priority={:?}", - rule.tool_name, - rule.priority - ); - // The stored copy (with `created_at` preserved and `updated_at` - // refreshed) is discarded: the contract returns unit, and re-reading it - // is `tool_rules`' job. - self.tool_memory() - .await? - .put_rule(rule) - .await - .map(|_| ()) - .map_err(classify_put_rule) - } - - async fn delete_tool_rule(&self, tool_name: &str, rule_id: &str) -> Result { - log::debug!("[memory:driver:embedded] delete_tool_rule tool={tool_name} rule={rule_id}"); - self.tool_memory() - .await? - .delete_rule(tool_name, rule_id) - .await - .map_err(|error| host_error("delete_tool_rule", error)) - } -} - -#[cfg(test)] -#[path = "tool_memory_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/tool_memory_tests.rs b/src/openhuman/memory/driver/embedded/tool_memory_tests.rs deleted file mode 100644 index c2f610a8d5..0000000000 --- a/src/openhuman/memory/driver/embedded/tool_memory_tests.rs +++ /dev/null @@ -1,197 +0,0 @@ -//! [`MemoryToolMemory`] tests. -//! -//! Two carry weight beyond a round-trip: -//! -//! - `put_tool_rule_with_a_blank_tool_name_is_invalid_not_other` pins the error -//! classification. `ToolMemoryStore::put_rule` rejects a blank name before -//! touching storage; collapsing that into `Other` would tell a caller their -//! backend is broken when their input is. -//! - `put_tool_rule_through_the_contract_is_visible_to_an_independent_reader` -//! is the same-store proof: a second client built over the same workspace -//! sees the rule, so the contract write reached the workspace's real store -//! rather than something private to the driver's handle. - -use super::super::test_support::fresh_driver; -use super::*; - -use crate::openhuman::config::schema::MemoryHooksConfig; -use crate::openhuman::memory::driver::embedded::EmbeddedMemoryProvider; -use crate::openhuman::memory::tool_memory::{ToolMemoryPriority, ToolMemorySource}; - -fn rule(tool: &str, body: &str, priority: ToolMemoryPriority) -> ToolMemoryRule { - ToolMemoryRule::new(tool, body, priority, ToolMemorySource::Programmatic) -} - -#[tokio::test] -async fn put_tool_rule_then_tool_rules_returns_it() { - let (_tmp, provider) = fresh_driver(); - - let stored = rule("shell", "never rm -rf /", ToolMemoryPriority::Critical); - provider - .put_tool_rule(stored.clone()) - .await - .expect("put_tool_rule"); - - let rules = provider.tool_rules("shell").await.expect("tool_rules"); - assert_eq!(rules.len(), 1); - assert_eq!(rules[0].id, stored.id); - assert_eq!(rules[0].rule, "never rm -rf /"); - assert_eq!(rules[0].priority, ToolMemoryPriority::Critical); -} - -#[tokio::test] -async fn tool_rules_is_empty_for_a_tool_with_no_rules() { - let (_tmp, provider) = fresh_driver(); - assert!(provider - .tool_rules("never-used") - .await - .expect("tool_rules") - .is_empty()); -} - -#[tokio::test] -async fn tool_rules_orders_critical_before_high_before_normal() { - let (_tmp, provider) = fresh_driver(); - - for (body, priority) in [ - ("normal one", ToolMemoryPriority::Normal), - ("critical one", ToolMemoryPriority::Critical), - ("high one", ToolMemoryPriority::High), - ] { - provider - .put_tool_rule(rule("email", body, priority)) - .await - .expect("put_tool_rule"); - } - - let rules = provider.tool_rules("email").await.expect("tool_rules"); - let priorities: Vec = rules.iter().map(|r| r.priority).collect(); - assert_eq!( - priorities, - vec![ - ToolMemoryPriority::Critical, - ToolMemoryPriority::High, - ToolMemoryPriority::Normal - ], - "the contract says highest priority first" - ); -} - -#[tokio::test] -async fn put_tool_rule_upserts_on_the_same_id() { - let (_tmp, provider) = fresh_driver(); - - let mut existing = rule("shell", "first body", ToolMemoryPriority::Normal); - provider - .put_tool_rule(existing.clone()) - .await - .expect("first put"); - existing.rule = "second body".to_string(); - provider - .put_tool_rule(existing.clone()) - .await - .expect("second put"); - - let rules = provider.tool_rules("shell").await.expect("tool_rules"); - assert_eq!(rules.len(), 1, "same id must upsert, not duplicate"); - assert_eq!(rules[0].rule, "second body"); -} - -#[tokio::test] -async fn put_tool_rule_with_a_blank_tool_name_is_invalid_not_other() { - let (_tmp, provider) = fresh_driver(); - - let mut blank = rule("shell", "some body", ToolMemoryPriority::Normal); - blank.tool_name = " ".to_string(); - - let error = provider - .put_tool_rule(blank) - .await - .expect_err("a blank tool name must be rejected"); - assert!( - matches!(error, MemoryError::Invalid(_)), - "caller error must not be reported as a backend failure: {error:?}" - ); -} - -#[tokio::test] -async fn put_tool_rule_with_a_blank_body_is_invalid_not_other() { - let (_tmp, provider) = fresh_driver(); - - let mut blank = rule("shell", "placeholder", ToolMemoryPriority::Normal); - blank.rule = " ".to_string(); - - let error = provider - .put_tool_rule(blank) - .await - .expect_err("a blank rule body must be rejected"); - assert!(matches!(error, MemoryError::Invalid(_)), "{error:?}"); -} - -#[tokio::test] -async fn delete_tool_rule_reports_existence_then_is_idempotent() { - let (_tmp, provider) = fresh_driver(); - - let stored = rule("shell", "a rule", ToolMemoryPriority::Normal); - provider - .put_tool_rule(stored.clone()) - .await - .expect("put_tool_rule"); - - assert!( - provider - .delete_tool_rule("shell", &stored.id) - .await - .expect("first delete"), - "the first delete must report that the rule existed" - ); - assert!( - !provider - .delete_tool_rule("shell", &stored.id) - .await - .expect("second delete"), - "deleting twice is a successful no-op, not an error" - ); - assert!(provider - .tool_rules("shell") - .await - .expect("tool_rules") - .is_empty()); -} - -#[tokio::test] -async fn put_tool_rule_through_the_contract_is_visible_to_an_independent_reader() { - use crate::openhuman::memory::store::MemoryClient; - use crate::openhuman::memory::tool_memory::tool_memory_store; - - let tmp = tempfile::TempDir::new().expect("temp workspace"); - let workspace = tmp.path().join("ws"); - let provider = EmbeddedMemoryProvider::new(workspace.clone(), MemoryHooksConfig::default()); - - let stored = rule( - "driver_store_proof", - "reachable from a second handle", - ToolMemoryPriority::High, - ); - provider - .put_tool_rule(stored.clone()) - .await - .expect("put_tool_rule"); - - // A second, independently constructed client over the same workspace — - // exactly how `memory::ops::tool_memory::open_store` builds its store, but - // without the process-global slot. The RPC handler itself resolves that - // global, which any concurrently running test may rebind to its own - // workspace mid-body; dialling it here would make this proof flaky for a - // reason that has nothing to do with the driver. - let independent = MemoryClient::from_workspace_dir(workspace).expect("second client"); - let rules = tool_memory_store(independent.memory_handle()) - .list_rules("driver_store_proof") - .await - .expect("list_rules"); - - assert!( - rules.iter().any(|r| r.id == stored.id), - "contract write must land in the workspace store every other reader sees: {rules:?}" - ); -} diff --git a/src/openhuman/memory/driver/embedded/tree.rs b/src/openhuman/memory/driver/embedded/tree.rs deleted file mode 100644 index e6472cfef8..0000000000 --- a/src/openhuman/memory/driver/embedded/tree.rs +++ /dev/null @@ -1,259 +0,0 @@ -//! [`MemoryTree`] for the embedded driver — the markdown time-summary tree. -//! -//! ## The family is `tree_runtime`, not `tree::retrieval` -//! -//! This is worth stating up front because the obvious reading of the method -//! names points at the wrong module. `tree::retrieval::{query_source, -//! drill_down}` are *retrieval* entry points: they return `QueryResponse<…>` / -//! `Vec` over the hybrid ranker, and the `TreeStatus` in -//! `memory::store::trees::types` is a **different type with the same name** (an -//! enum with an `Active` variant, describing a sealed source tree). -//! -//! The contract's [`IngestRequest`], [`QueryResult`], [`TreeNode`] and -//! [`TreeStatus`] are the *runtime* tree's types — literally so: -//! `memory::tree::tree_runtime` re-exports -//! `tinycortex::memory::tree::runtime::*`, and that module in turn is -//! `pub use tinycortex_api::tree as types`. Contract type and host type are -//! **the same type**, so three of the five methods below convert nothing. -//! -//! ## The source-scope hazard does not arise here -//! -//! The M3c brief warned that threading `scope` into `retrieval::query_source` -//! would apply the allowlist twice — once as the explicit parameter and once -//! through the `current_source_scope()` task-local that callee reads -//! internally. That warning is real, and this file avoids it by not going -//! there: [`Self::query_source`] reads the **chunk store** -//! (`store::chunks::store::list_chunks`), whose `ListChunksQuery.source_scope` -//! is already an explicit parameter applied **in SQL before `LIMIT`**. -//! -//! That SQL predicate is predicate 3 of the three pinned by -//! `tree::retrieval::source_scope_tests`, and it is the one -//! [`SourceScope::allows_source_id`] was written against — equality or -//! `mem_src:{allowed}:` prefix, untagged content fails open, an empty allow -//! list keeps only untagged rows. So no predicate changes, no task-local is -//! read, and all 25 characterization tests stay untouched. -//! -//! ## `namespace` has no home on the chunk tier -//! -//! `mem_tree_chunks` has no namespace column — chunks are keyed by -//! `(source_kind, source_id)`. [`Self::query_source`] therefore *validates* -//! `namespace` (so a traversal attempt is still refused) and otherwise ignores -//! it. Said out loud rather than dropped silently. -//! -//! ## Sealing needs a summarisation model -//! -//! [`Self::seal`] and [`Self::cascade`] drive the LLM fold, so they resolve a -//! provider through `tree_runtime::ops::create_provider` — the same resolver -//! the `tree_summarizer.*` RPC path and the memory doctor use. When the host -//! has neither local AI nor `memory_tree.cloud_summarization_opt_in`, that -//! resolver fails and the call surfaces as [`MemoryError::Invalid`] carrying -//! the existing operator-facing message. -//! -//! Both short-circuit when there is nothing to do — an empty buffer for -//! `seal`, an empty tree for `cascade` — and return the current status without -//! resolving a provider at all. The contract requires both to be idempotent -//! no-ops in exactly those cases, and a no-op must not need a model. - -use async_trait::async_trait; -use std::collections::HashSet; -use tinycortex_api::chunks::Chunk; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::SourceScope; -use tinycortex_api::provider::MemoryTree; -use tinycortex_api::tree::{IngestRequest, QueryResult, TreeStatus}; - -use crate::openhuman::config::Config; -use crate::openhuman::memory::store::chunks::store::{list_chunks, ListChunksQuery}; -use crate::openhuman::memory::tree::tree_runtime::{engine, ops, store}; - -use super::{host_error, EmbeddedMemoryProvider}; - -/// Runs a blocking store call on the blocking pool with an owned `Config`. -/// -/// Every `tree_runtime::store` and `chunks::store` entry point is synchronous -/// and hits SQLite or the filesystem; calling one straight from an async -/// contract method would stall the reactor. -async fn blocking(config: &Config, context: &'static str, run: F) -> Result -where - T: Send + 'static, - F: FnOnce(&Config) -> anyhow::Result + Send + 'static, -{ - let config = config.clone(); - tokio::task::spawn_blocking(move || run(&config)) - .await - .map_err(|error| host_error(context, format!("join error: {error}")))? - .map_err(|error| host_error(context, format!("{error:#}"))) -} - -/// `validate_namespace` / `validate_node_id` failures are caller errors, so -/// they become [`MemoryError::Invalid`] rather than the opaque `Other` the -/// host's `String` channel would otherwise collapse to. -fn invalid(reason: String) -> MemoryError { - MemoryError::Invalid(reason) -} - -#[async_trait] -impl MemoryTree for EmbeddedMemoryProvider { - async fn append(&self, request: IngestRequest) -> Result<(), MemoryError> { - log::debug!( - "[memory:driver:embedded] tree_append namespace={} content_chars={} has_metadata={}", - request.namespace, - request.content.chars().count(), - request.metadata.is_some() - ); - store::validate_namespace(&request.namespace).map_err(invalid)?; - if request.content.trim().is_empty() { - return Err(invalid("content must not be empty".to_string())); - } - - let config = self.config().await?; - // Mirrors `ops::tree_summarizer_ingest` exactly: trimmed namespace, - // ingest-time fallback for the timestamp. The returned buffer path is - // an implementation detail and is dropped. - let namespace = request.namespace.trim().to_string(); - let timestamp = request.timestamp.unwrap_or_else(chrono::Utc::now); - let content = request.content; - let metadata = request.metadata; - blocking(config, "tree_append", move |config| { - store::buffer_write(config, &namespace, &content, ×tamp, metadata.as_ref()) - .map(|_path| ()) - }) - .await - } - - async fn query_source( - &self, - namespace: &str, - source_id: &str, - limit: usize, - scope: Option<&SourceScope>, - ) -> Result, MemoryError> { - log::debug!( - "[memory:driver:embedded] tree_query_source namespace={namespace} \ - source_id={source_id} limit={limit} scoped={}", - scope.is_some() - ); - // Validated but not used as a filter — see the module docs. - store::validate_namespace(namespace).map_err(invalid)?; - - let query = ListChunksQuery { - source_id: Some(source_id.to_string()), - // The allowlist travels into SQL, applied before `LIMIT`. This is - // the whole point of the contract taking `scope` as a parameter. - source_scope: scope - .map(|scope| scope.allow.iter().cloned().collect::>()), - limit: Some(limit), - exclude_dropped: true, - ..ListChunksQuery::default() - }; - - let config = self.config().await?; - // `ORDER BY timestamp_ms DESC` in `list_chunks` is the contract's - // "newest first"; no re-sorting here. - blocking(config, "tree_query_source", move |config| { - list_chunks(config, &query) - }) - .await - } - - async fn drill_down(&self, namespace: &str, node_id: &str) -> Result { - log::debug!( - "[memory:driver:embedded] tree_drill_down namespace={namespace} node_id={node_id}" - ); - store::validate_namespace(namespace).map_err(invalid)?; - store::validate_node_id(node_id).map_err(invalid)?; - - let config = self.config().await?; - let namespace = namespace.trim().to_string(); - let node_id = node_id.to_string(); - let found = { - let namespace = namespace.clone(); - let node_id = node_id.clone(); - blocking(config, "tree_drill_down", move |config| { - let Some(node) = store::read_node(config, &namespace, &node_id)? else { - return Ok(None); - }; - let children = store::read_children(config, &namespace, &node_id)?; - Ok(Some(QueryResult { node, children })) - }) - .await? - }; - - // The contract mandates `NotFound` here. The RPC path returns a - // `String` for the same case, which is why this is constructed in the - // driver rather than mapped from below. - found.ok_or_else(|| { - MemoryError::NotFound(format!("tree node '{node_id}' not found in '{namespace}'")) - }) - } - - async fn seal(&self, namespace: &str) -> Result { - log::debug!("[memory:driver:embedded] tree_seal namespace={namespace}"); - store::validate_namespace(namespace).map_err(invalid)?; - let config = self.config().await?; - let namespace = namespace.trim().to_string(); - - // Nothing buffered ⇒ nothing to seal. Short-circuited *before* the - // provider is resolved so a scheduler may call `seal` unconditionally - // on a host with no summarisation model without seeing an error. - let buffered = { - let namespace = namespace.clone(); - blocking(config, "tree_seal_buffer_read", move |config| { - store::buffer_read(config, &namespace) - }) - .await? - }; - if !buffered.is_empty() { - let provider = ops::create_provider(config) - .map_err(invalid) - .map(|(provider, _model)| provider)?; - // `Ok(None)` means the buffer emptied under us — still a success. - engine::run_summarization(config, provider.as_ref(), &namespace, chrono::Utc::now()) - .await - .map_err(|error| host_error("tree_seal", format!("{error:#}")))?; - } - - blocking(config, "tree_seal_status", move |config| { - store::get_tree_status(config, &namespace) - }) - .await - } - - async fn cascade(&self, namespace: &str) -> Result { - log::debug!("[memory:driver:embedded] tree_cascade namespace={namespace}"); - store::validate_namespace(namespace).map_err(invalid)?; - let config = self.config().await?; - let namespace = namespace.trim().to_string(); - - // An empty tree has no leaves to roll up. Same short-circuit rationale - // as `seal`. - let status = { - let namespace = namespace.clone(); - blocking(config, "tree_cascade_status", move |config| { - store::get_tree_status(config, &namespace) - }) - .await? - }; - if status.total_nodes == 0 { - return Ok(status); - } - - let provider = ops::create_provider(config) - .map_err(invalid) - .map(|(provider, _model)| provider)?; - // NOTE: `rebuild_tree` recomputes every parent level from the hour - // leaves rather than incrementally rolling up only what changed. Same - // direction and same resulting state as the contract's "roll sealed - // leaves up through the parent levels", and idempotent as required — - // but more expensive than the word "cascade" suggests. There is no - // incremental host entry point to delegate to, and writing one would - // be engine logic. - engine::rebuild_tree(config, provider.as_ref(), &namespace) - .await - .map_err(|error| host_error("tree_cascade", format!("{error:#}"))) - } -} - -#[cfg(test)] -#[path = "tree_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/embedded/tree_tests.rs b/src/openhuman/memory/driver/embedded/tree_tests.rs deleted file mode 100644 index a1495b09e1..0000000000 --- a/src/openhuman/memory/driver/embedded/tree_tests.rs +++ /dev/null @@ -1,378 +0,0 @@ -//! [`MemoryTree`] tests for the embedded driver. -//! -//! The scope tests below deliberately mirror -//! `tree::retrieval::source_scope_tests`' predicate-3 cases, but reach them -//! *through the driver*. Those 25 characterization tests keep asserting the -//! host predicate directly and are untouched; these assert that routing a -//! contract `SourceScope` into `ListChunksQuery.source_scope` preserves it. - -use super::super::test_support::fresh_driver; - -use chrono::{TimeZone, Utc}; -use tinycortex_api::provider::types::SourceScope; -use tinycortex_api::provider::MemoryTree; -use tinycortex_api::tree::IngestRequest; - -use crate::openhuman::config::Config; -use crate::openhuman::memory::store::chunks::store::{ - upsert_chunks, upsert_staged_chunks_tx, with_connection, -}; -use crate::openhuman::memory::store::chunks::types::{ - chunk_id, Chunk, Metadata, SourceKind, SourceRef, -}; -use crate::openhuman::memory::store::content as content_store; - -const BASE_MS: i64 = 1_700_000_000_000; -const MEMORY_SOURCES: &str = "memory_sources"; - -fn request(namespace: &str, content: &str) -> IngestRequest { - IngestRequest { - namespace: namespace.to_string(), - content: content.to_string(), - timestamp: Some(Utc.timestamp_millis_opt(BASE_MS).unwrap()), - metadata: None, - } -} - -/// A chunk in `source`, tagged with `tags`, timestamped `ts_ms`. Same shape as -/// the `source_scope_tests` fixture so the two suites stay comparable. -fn src_chunk(source: &str, seq: u32, tags: &[&str], ts_ms: i64) -> Chunk { - let ts = Utc.timestamp_millis_opt(ts_ms).unwrap(); - Chunk { - id: chunk_id(SourceKind::Chat, source, seq, "driver-content"), - content: format!("content-{source}-{seq}"), - metadata: Metadata { - source_kind: SourceKind::Chat, - source_id: source.into(), - owner: "alice".into(), - timestamp: ts, - time_range: (ts, ts), - tags: tags.iter().map(|t| (*t).to_string()).collect(), - source_ref: Some(SourceRef::new(format!("slack://{source}/{seq}"))), - path_scope: None, - }, - token_count: 20, - seq_in_source: seq, - created_at: ts, - partial_message: false, - } -} - -fn seed_chunks(config: &Config, chunks: &[Chunk]) { - upsert_chunks(config, chunks).expect("upsert_chunks"); - let content_root = config.memory_tree_content_root(); - std::fs::create_dir_all(&content_root).expect("create content_root"); - let staged = content_store::stage_chunks(&content_root, chunks).expect("stage_chunks"); - with_connection(config, |conn| { - let tx = conn.unchecked_transaction()?; - upsert_staged_chunks_tx(&tx, &staged)?; - tx.commit()?; - Ok(()) - }) - .expect("persist staged chunk pointers"); -} - -// ── append ─────────────────────────────────────────────────────────────── - -#[tokio::test] -async fn tree_append_buffers_content_for_namespace() { - let (_tmp, provider) = fresh_driver(); - provider - .append(request("work", "phoenix launch is friday")) - .await - .expect("append"); - - let config = provider.config().await.expect("config"); - let buffered = crate::openhuman::memory::tree::tree_runtime::store::buffer_read(config, "work") - .expect("buffer_read"); - assert_eq!(buffered.len(), 1, "one buffered entry"); - assert!( - buffered[0].1.contains("phoenix launch is friday"), - "buffered body must carry the content, got {:?}", - buffered[0].1 - ); -} - -#[tokio::test] -async fn tree_append_rejects_empty_content_as_invalid() { - let (_tmp, provider) = fresh_driver(); - let error = provider - .append(request("work", " \n ")) - .await - .expect_err("whitespace-only content must be refused"); - assert!( - matches!(error, tinycortex_api::error::MemoryError::Invalid(_)), - "expected Invalid, got {error:?}" - ); -} - -#[tokio::test] -async fn tree_append_rejects_traversing_namespace_as_invalid() { - let (_tmp, provider) = fresh_driver(); - let error = provider - .append(request("../escape", "body")) - .await - .expect_err("traversal namespace must be refused"); - assert!( - matches!(error, tinycortex_api::error::MemoryError::Invalid(_)), - "expected Invalid, got {error:?}" - ); -} - -// ── drill_down ─────────────────────────────────────────────────────────── - -#[tokio::test] -async fn tree_drill_down_unknown_node_is_not_found() { - let (_tmp, provider) = fresh_driver(); - let error = provider - .drill_down("work", "2024/03/15/09") - .await - .expect_err("an absent node must not be Ok"); - assert!( - matches!(error, tinycortex_api::error::MemoryError::NotFound(_)), - "the contract mandates NotFound here, got {error:?}" - ); -} - -#[tokio::test] -async fn tree_drill_down_returns_node_with_direct_children() { - use crate::openhuman::memory::tree::tree_runtime::store::write_node; - use tinycortex::memory::tree::runtime::{NodeLevel, TreeNode}; - - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - let ts = Utc.timestamp_millis_opt(BASE_MS).unwrap(); - - let node = |node_id: &str, level: NodeLevel, parent: Option<&str>| TreeNode { - node_id: node_id.to_string(), - namespace: "work".to_string(), - level, - parent_id: parent.map(str::to_string), - summary: format!("summary for {node_id}"), - token_count: 10, - child_count: 0, - created_at: ts, - updated_at: ts, - metadata: None, - }; - - write_node(&config, &node("2024", NodeLevel::Year, Some("root"))).expect("write year"); - write_node(&config, &node("2024/03", NodeLevel::Month, Some("2024"))).expect("write month"); - - let result = provider - .drill_down("work", "2024") - .await - .expect("drill_down"); - assert_eq!(result.node.node_id, "2024"); - assert_eq!( - result - .children - .iter() - .map(|child| child.node_id.as_str()) - .collect::>(), - vec!["2024/03"], - ); -} - -#[tokio::test] -async fn tree_drill_down_rejects_traversing_node_id_as_invalid() { - let (_tmp, provider) = fresh_driver(); - let error = provider - .drill_down("work", "../../etc") - .await - .expect_err("traversal node id must be refused"); - assert!( - matches!(error, tinycortex_api::error::MemoryError::Invalid(_)), - "expected Invalid, got {error:?}" - ); -} - -// ── query_source + scope ───────────────────────────────────────────────── - -#[tokio::test] -async fn tree_query_source_returns_that_sources_chunks_newest_first() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - seed_chunks( - &config, - &[ - src_chunk("src-abc", 1, &[], BASE_MS), - src_chunk("src-abc", 2, &[], BASE_MS + 1_000), - src_chunk("src-xyz", 1, &[], BASE_MS + 2_000), - ], - ); - - let hits = provider - .query_source("work", "src-abc", 10, None) - .await - .expect("query_source"); - - assert_eq!(hits.len(), 2, "only src-abc's chunks"); - assert!( - hits[0].metadata.timestamp >= hits[1].metadata.timestamp, - "newest first" - ); -} - -#[tokio::test] -async fn tree_query_source_unknown_source_is_empty_not_an_error() { - let (_tmp, provider) = fresh_driver(); - let hits = provider - .query_source("work", "src-nope", 10, None) - .await - .expect("an unknown source must yield an empty vector, not an error"); - assert!(hits.is_empty()); -} - -#[tokio::test] -async fn tree_query_source_scope_admits_the_listed_source() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - seed_chunks( - &config, - &[src_chunk("src-abc", 1, &[MEMORY_SOURCES], BASE_MS)], - ); - - let scope = SourceScope::new(["src-abc"]); - let hits = provider - .query_source("work", "src-abc", 10, Some(&scope)) - .await - .expect("query_source"); - assert_eq!(hits.len(), 1); - - let other = SourceScope::new(["src-other"]); - let hits = provider - .query_source("work", "src-abc", 10, Some(&other)) - .await - .expect("query_source"); - assert!( - hits.is_empty(), - "a source-tagged chunk outside scope is denied" - ); -} - -#[tokio::test] -async fn tree_query_source_scope_admits_mem_src_prefix() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - seed_chunks( - &config, - &[src_chunk( - "mem_src:src-abc:item-1", - 1, - &[MEMORY_SOURCES], - BASE_MS, - )], - ); - - let scope = SourceScope::new(["src-abc"]); - let hits = provider - .query_source("work", "mem_src:src-abc:item-1", 10, Some(&scope)) - .await - .expect("query_source"); - assert_eq!( - hits.len(), - 1, - "the `mem_src:{{allowed}}:` prefix rule must survive the driver hop" - ); -} - -#[tokio::test] -async fn tree_query_source_empty_scope_keeps_only_untagged_chunks() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - seed_chunks( - &config, - &[ - src_chunk("src-abc", 1, &[MEMORY_SOURCES], BASE_MS), - src_chunk("src-plain", 1, &[], BASE_MS), - ], - ); - - let empty = SourceScope::default(); - assert!( - provider - .query_source("work", "src-abc", 10, Some(&empty)) - .await - .expect("query_source") - .is_empty(), - "an empty allow list denies all source-attributed content" - ); - assert_eq!( - provider - .query_source("work", "src-plain", 10, Some(&empty)) - .await - .expect("query_source") - .len(), - 1, - "untagged content fails open, exactly as the SQL predicate does" - ); -} - -#[tokio::test] -async fn tree_query_source_scope_is_applied_before_limit() { - let (_tmp, provider) = fresh_driver(); - let config = provider.config().await.expect("config").clone(); - // Two out-of-scope chunks are NEWER than the in-scope one. A post-filter - // would spend the limit on them and return nothing; a SQL predicate before - // LIMIT returns the in-scope row. - seed_chunks( - &config, - &[ - src_chunk("src-abc", 1, &[MEMORY_SOURCES], BASE_MS), - src_chunk("src-abc", 2, &[MEMORY_SOURCES], BASE_MS + 1_000), - src_chunk("src-abc", 3, &[MEMORY_SOURCES], BASE_MS + 2_000), - ], - ); - - let scope = SourceScope::new(["src-abc"]); - let hits = provider - .query_source("work", "src-abc", 1, Some(&scope)) - .await - .expect("query_source"); - assert_eq!(hits.len(), 1, "limit is honoured"); -} - -// ── seal / cascade ─────────────────────────────────────────────────────── - -#[tokio::test] -async fn tree_seal_on_empty_buffer_is_a_successful_noop() { - let (_tmp, provider) = fresh_driver(); - // The default test config resolves NO summarisation provider. Sealing - // nothing must still succeed, which is why the empty-buffer check runs - // before provider resolution. - let status = provider.seal("work").await.expect("seal an empty buffer"); - assert_eq!(status.namespace, "work"); - assert_eq!(status.total_nodes, 0); -} - -#[tokio::test] -async fn tree_seal_with_buffered_content_needs_a_summarization_provider() { - let (_tmp, provider) = fresh_driver(); - provider - .append(request("work", "something to summarise")) - .await - .expect("append"); - - let error = provider - .seal("work") - .await - .expect_err("no local AI and no cloud opt-in means no provider"); - match error { - tinycortex_api::error::MemoryError::Invalid(reason) => assert!( - reason.contains("summarization provider"), - "the operator-facing resolver message must survive, got {reason}" - ), - other => panic!("expected Invalid, got {other:?}"), - } -} - -#[tokio::test] -async fn tree_cascade_on_empty_tree_is_a_successful_noop() { - let (_tmp, provider) = fresh_driver(); - let status = provider - .cascade("work") - .await - .expect("cascade an empty tree"); - assert_eq!(status.total_nodes, 0); -} diff --git a/src/openhuman/memory/driver/mod.rs b/src/openhuman/memory/driver/mod.rs index ef2f3d59f3..5bd670677d 100644 --- a/src/openhuman/memory/driver/mod.rs +++ b/src/openhuman/memory/driver/mod.rs @@ -1,13 +1,5 @@ -//! Memory-driver implementations of the [`tinycortex_api`] contract. +//! Memory-driver namespace. //! -//! One subdirectory per driver. Today there is exactly one — [`embedded`], -//! which wraps the in-process tinycortex engine — plus the reference -//! `NullMemoryProvider` that ships inside the contract crate itself. -//! -//! Drivers live *under* `memory/` rather than in a sibling top-level directory -//! so the "one directory equals one feature gate" family rule holds: a memory -//! driver is memory, and gating it separately from the domain it implements -//! would be meaningless. - -pub mod embedded; -pub mod module_adapter; +//! Memory execution is provided by the compiled TinyMemory TinyBus module. +//! The host-side contract and binding live in [`super::api`] and +//! [`super::binding`]; there is intentionally no in-process engine driver here. diff --git a/src/openhuman/memory/driver/module_adapter.rs b/src/openhuman/memory/driver/module_adapter.rs deleted file mode 100644 index c31491cdfb..0000000000 --- a/src/openhuman/memory/driver/module_adapter.rs +++ /dev/null @@ -1,294 +0,0 @@ -//! Bridges a TinyMemory-contract driver into the TinyCortex-contract slot. -//! -//! `memory::binding::build` produces an `Arc`, while `modules::memory::ModuleMemoryProvider` implements -//! `tinymemory_api::provider::MemoryProvider`. Those are two distinct traits in -//! two distinct crates. This adapter is what lets the module-backed driver bind -//! before the host's own memory binding finishes migrating onto the TinyMemory -//! contract. -//! -//! # It exists to be deleted -//! -//! `tinymemory-api` was moved *out of* `tinycortex-api`, so the two describe the -//! same values with the same trait shape — `MemoryProvider: MemoryCore + -//! MemoryRecall + MemoryPortability` on both sides, with matching method -//! signatures. When the binding migrates, this file goes away and -//! `ModuleMemoryProvider` is bound directly. -//! -//! # Why serde round-trips and not field-by-field conversion -//! -//! This is the one decision here worth arguing with, so the argument is written -//! down. -//! -//! `tinymemory-tinycortex::convert` does it properly: exhaustive destructuring, -//! total matches, no `..`, so a field added to one contract is a compile error -//! rather than a silently dropped value. That discipline is right for a permanent -//! seam, and it is why the two conversions this adapter needed most -//! (`entry_to_tinycortex`, `namespace_summary_to_tinycortex`) were added there. -//! -//! The remaining types — `ExportPage`, `ExportRecord`, `ImportOutcome`, -//! `SourceScope`, `OwnedRecallOpts` — cross here by serde instead, and that is a -//! deliberate trade with a real downside: a round trip **tolerates** drift where -//! destructuring would catch it. A field added to one side and not the other is -//! dropped at runtime, silently. -//! -//! Three things make that acceptable *for this file specifically*: -//! -//! 1. The contracts are byte-identical today by construction, not by coincidence. -//! 2. [`round_trip_preserves_every_field`] fails if a value stops surviving the -//! crossing, which is the failure mode being traded away — so it is detected, -//! just at test time rather than compile time. -//! 3. This code has a scheduled death. Writing ~250 lines of exhaustive -//! conversion for a seam that is removed once the binding migrates would be -//! work done twice. -//! -//! Do **not** copy this approach into `convert.rs`, and do not extend this file -//! to carry a permanent conversion. If it stops being temporary, replace the -//! round trips with destructuring first. - -use std::sync::Arc; - -use async_trait::async_trait; -use tinycortex_api::capabilities::Capabilities as TcCapabilities; -use tinycortex_api::error::MemoryError as TcError; -use tinycortex_api::health::MemoryHealth as TcHealth; -use tinycortex_api::provider::types::{ - ExportPage as TcExportPage, ExportRecord as TcExportRecord, ImportOutcome as TcImportOutcome, - SourceScope as TcSourceScope, -}; -use tinycortex_api::provider::{ - MemoryCore as TcCore, MemoryPortability as TcPortability, MemoryProvider as TcProvider, - MemoryRecall as TcRecall, -}; -use tinycortex_api::recall::OwnedRecallOpts as TcRecallOpts; -use tinycortex_api::types::{ - MemoryCategory as TcCategory, MemoryEntry as TcEntry, MemoryTaint as TcTaint, - NamespaceSummary as TcSummary, -}; -use tinymemory_api::provider::MemoryProvider as TmProvider; -use tinymemory_tinycortex::convert; - -/// A TinyMemory driver, presented as a TinyCortex one. -pub struct TinyMemoryContractAdapter { - inner: Arc, -} - -impl std::fmt::Debug for TinyMemoryContractAdapter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TinyMemoryContractAdapter") - .field("driver_id", &self.inner.driver_id()) - .finish_non_exhaustive() - } -} - -impl TinyMemoryContractAdapter { - /// Wrap `inner` so it can bind in a TinyCortex-contract slot. - #[must_use] - pub fn new(inner: Arc) -> Self { - Self { inner } - } -} - -/// Cross a value between the two contracts through its serde form. -/// -/// # Errors -/// -/// A message naming the type, when the two contracts have drifted far enough -/// that a value no longer decodes. Never carries the value itself — these are -/// user memory records. -fn cross( - value: &A, - what: &'static str, -) -> Result { - let json = serde_json::to_value(value) - .map_err(|error| TcError::Other(anyhow::anyhow!("encode {what}: {error}")))?; - serde_json::from_value(json) - .map_err(|error| TcError::Other(anyhow::anyhow!("decode {what}: {error}"))) -} - -/// Map a TinyMemory error onto the TinyCortex one, preserving the variant. -/// -/// Variant-preserving rather than collapsing onto `Other`, because a host -/// re-raises these to its own callers and `NotFound` versus `Invalid` is -/// observable — `get`'s contract makes a miss `Ok(None)` while an `Invalid` is a -/// real failure. `Io` and `Serde` degrade to `Other`: neither foreign error can -/// be rebuilt, and inventing an `io::ErrorKind` would be worse than being honest -/// that this crossed a boundary. -fn error_to_tinycortex(error: tinymemory_api::error::MemoryError) -> TcError { - use tinymemory_api::error::MemoryError as Tm; - match error { - Tm::NotFound(message) => TcError::NotFound(message), - Tm::Invalid(message) => TcError::Invalid(message), - Tm::BudgetExceeded(message) => TcError::BudgetExceeded(message), - Tm::PathEscape(message) => TcError::PathEscape(message), - Tm::Unsupported { capability } => TcError::unsupported_raw(capability), - Tm::Io(inner) => TcError::Other(anyhow::anyhow!("io error: {inner}")), - Tm::Serde(inner) => TcError::Other(anyhow::anyhow!("serde error: {inner}")), - Tm::Other(inner) => TcError::Other(inner), - } -} - -#[async_trait] -impl TcProvider for TinyMemoryContractAdapter { - fn driver_id(&self) -> &str { - self.inner.driver_id() - } - - fn capabilities(&self) -> TcCapabilities { - // Both sides serialize a capability set as a JSON array of family names, - // so this crossing is over stable wire strings rather than bit layouts. - // An unreadable set is reported as empty, which is the fail-closed - // direction: the kernel filters its RPC surface from this, so an - // overstated set would register methods that answer errors. - cross::<_, TcCapabilities>(&self.inner.capabilities(), "capabilities") - .unwrap_or_else(|_| TcCapabilities::empty()) - } - - async fn health(&self) -> TcHealth { - let health = self.inner.health().await; - cross::<_, TcHealth>(&health, "health") - .unwrap_or_else(|error| TcHealth::down(error.to_string())) - } - - async fn shutdown(&self) -> Result<(), TcError> { - self.inner.shutdown().await.map_err(error_to_tinycortex) - } -} - -#[async_trait] -impl TcCore for TinyMemoryContractAdapter { - async fn store( - &self, - namespace: &str, - key: &str, - content: &str, - category: TcCategory, - session_id: Option<&str>, - taint: TcTaint, - ) -> Result<(), TcError> { - // Category and taint use the audited destructuring conversions, not a - // serde round trip. Taint especially: mapping it wrongly would let - // externally-sourced content be treated as internal-trust content, which - // is the one thing the policy guard exists to prevent. - self.inner - .store( - namespace, - key, - content, - convert::category_to_tinymemory(category), - session_id, - convert::taint_to_tinymemory(taint), - ) - .await - .map_err(error_to_tinycortex) - } - - async fn get(&self, namespace: &str, key: &str) -> Result, TcError> { - Ok(self - .inner - .get(namespace, key) - .await - .map_err(error_to_tinycortex)? - .map(convert::entry_to_tinycortex)) - } - - async fn forget(&self, namespace: &str, key: &str) -> Result { - self.inner - .forget(namespace, key) - .await - .map_err(error_to_tinycortex) - } - - async fn list( - &self, - namespace: Option<&str>, - category: Option<&TcCategory>, - session_id: Option<&str>, - ) -> Result, TcError> { - let category = category.cloned().map(convert::category_to_tinymemory); - Ok(self - .inner - .list(namespace, category.as_ref(), session_id) - .await - .map_err(error_to_tinycortex)? - .into_iter() - .map(convert::entry_to_tinycortex) - .collect()) - } - - async fn namespaces(&self) -> Result, TcError> { - Ok(self - .inner - .namespaces() - .await - .map_err(error_to_tinycortex)? - .into_iter() - .map(convert::namespace_summary_to_tinycortex) - .collect()) - } -} - -#[async_trait] -impl TcRecall for TinyMemoryContractAdapter { - async fn recall( - &self, - query: &str, - limit: usize, - opts: &TcRecallOpts, - scope: Option<&TcSourceScope>, - ) -> Result, TcError> { - let opts = cross::<_, tinymemory_api::recall::OwnedRecallOpts>(opts, "recall options")?; - // `scope` is a query predicate the driver applies internally, so it has - // to cross intact rather than being applied to the result here — see the - // contract's note on why filtering afterwards is wrong. - let scope = match scope { - Some(scope) => Some(cross::<_, tinymemory_api::provider::types::SourceScope>( - scope, - "source scope", - )?), - None => None, - }; - Ok(self - .inner - .recall(query, limit, &opts, scope.as_ref()) - .await - .map_err(error_to_tinycortex)? - .into_iter() - .map(convert::entry_to_tinycortex) - .collect()) - } -} - -#[async_trait] -impl TcPortability for TinyMemoryContractAdapter { - async fn export_page( - &self, - cursor: Option<&str>, - limit: usize, - ) -> Result { - let page = self - .inner - .export_page(cursor, limit) - .await - .map_err(error_to_tinycortex)?; - cross(&page, "export page") - } - - async fn import_records( - &self, - records: Vec, - ) -> Result { - let records: Vec = - cross(&records, "export records")?; - let outcome = self - .inner - .import_records(records) - .await - .map_err(error_to_tinycortex)?; - cross(&outcome, "import outcome") - } -} - -#[cfg(test)] -#[path = "module_adapter_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/driver/module_adapter_tests.rs b/src/openhuman/memory/driver/module_adapter_tests.rs deleted file mode 100644 index 98a35e55ae..0000000000 --- a/src/openhuman/memory/driver/module_adapter_tests.rs +++ /dev/null @@ -1,180 +0,0 @@ -//! The serde crossings are the risk this adapter takes on, so these tests are -//! about the crossings rather than about the forwarding. - -use super::{cross, error_to_tinycortex}; - -#[test] -fn round_trip_preserves_every_field() { - // This is the test the module docs point at. A serde round trip tolerates - // drift where exhaustive destructuring would catch it, so the trade is only - // acceptable while something checks that values still survive the crossing. - // - // Each case is populated with non-default values in every field it has, so a - // dropped field shows up as an inequality rather than as two defaults that - // happen to match. `..Default::default()` is deliberately not used here: - // a field left on its default would round-trip back to that same default - // even if the crossing dropped it, silently passing. - let opts = tinycortex_api::recall::OwnedRecallOpts { - namespace: Some("work".to_string()), - category: Some(tinycortex_api::types::MemoryCategory::Core), - session_id: Some("session-9".to_string()), - min_score: Some(0.42), - cross_session: true, - }; - let crossed: tinymemory_api::recall::OwnedRecallOpts = - cross(&opts, "recall options").expect("recall options cross"); - let back: tinycortex_api::recall::OwnedRecallOpts = - cross(&crossed, "recall options").expect("recall options cross back"); - assert_eq!(back, opts, "recall options lost a field crossing contracts"); -} - -#[test] -fn an_export_page_survives_the_crossing_with_its_cursor() { - // `next_cursor` is the terminator a caller keys on — an empty `records` is - // explicitly not one — so losing it would turn a paged export into an - // infinite loop or a silent truncation. - let page = tinycortex_api::provider::types::ExportPage { - records: Vec::new(), - next_cursor: Some("cursor-42".to_string()), - }; - let crossed: tinymemory_api::provider::types::ExportPage = - cross(&page, "export page").expect("cross"); - assert_eq!(crossed.next_cursor.as_deref(), Some("cursor-42")); - - let back: tinycortex_api::provider::types::ExportPage = - cross(&crossed, "export page").expect("cross back"); - assert_eq!(back, page); -} - -#[test] -fn an_export_record_survives_the_crossing_with_its_payload_and_taint() { - // `taint` and `payload` are exactly the fields a silent drop would corrupt - // worst: a dropped taint would let externally-sourced content re-enter as - // internal-trust, and a dropped payload would import an empty record. - let record = tinycortex_api::provider::types::ExportRecord { - kind: "entry".to_string(), - id: "entry-7".to_string(), - namespace: Some("work".to_string()), - taint: tinycortex_api::types::MemoryTaint::ExternalSync, - payload: serde_json::json!({"key": "value", "n": 3}), - }; - let crossed: tinymemory_api::provider::types::ExportRecord = - cross(&record, "export record").expect("cross"); - let back: tinycortex_api::provider::types::ExportRecord = - cross(&crossed, "export record").expect("cross back"); - assert_eq!( - back, record, - "export record lost a field crossing contracts" - ); -} - -#[test] -fn an_import_outcome_survives_the_crossing_with_its_counts_and_errors() { - // `errors` is operator-facing diagnosis; a dropped value here would leave - // a non-zero `failed` count with nothing to explain it. - let outcome = tinycortex_api::provider::types::ImportOutcome { - imported: 4, - skipped: 2, - failed: 1, - errors: vec!["record 'x' rejected: bad payload".to_string()], - }; - let crossed: tinymemory_api::provider::types::ImportOutcome = - cross(&outcome, "import outcome").expect("cross"); - let back: tinycortex_api::provider::types::ImportOutcome = - cross(&crossed, "import outcome").expect("cross back"); - assert_eq!( - back, outcome, - "import outcome lost a field crossing contracts" - ); -} - -#[test] -fn a_source_scope_survives_the_crossing_with_every_allowed_source() { - // An empty scope is fail-closed (denies all source-attributed content, per - // the type's own docs), so losing an entry here would silently widen what - // a recall call is allowed to see. - let scope = tinycortex_api::provider::types::SourceScope::new(["src-a", "src-b"]); - let crossed: tinymemory_api::provider::types::SourceScope = - cross(&scope, "source scope").expect("cross"); - let back: tinycortex_api::provider::types::SourceScope = - cross(&crossed, "source scope").expect("cross back"); - assert_eq!(back, scope, "source scope lost an entry crossing contracts"); -} - -#[test] -fn capabilities_cross_as_family_names_not_bit_layouts() { - // Both sides serialize a capability set as a JSON array of family names, so - // the crossing is over stable wire strings. If it went over the bitset an - // added family on one side would silently shift every bit above it. - let capabilities = tinymemory_api::capabilities::Capabilities::mandatory(); - let crossed: tinycortex_api::capabilities::Capabilities = - cross(&capabilities, "capabilities").expect("cross"); - - for mandatory in tinycortex_api::capabilities::Capability::MANDATORY { - assert!( - crossed.contains(mandatory), - "{mandatory:?} was lost crossing contracts" - ); - } -} - -#[test] -fn a_not_found_stays_not_found() { - // A host re-raises these to its own callers, and `get`'s contract makes a - // miss `Ok(None)` while an `Invalid` is a real failure — so collapsing the - // two is observable. - let error = error_to_tinycortex(tinymemory_api::error::MemoryError::NotFound( - "absent".to_string(), - )); - assert!( - matches!(error, tinycortex_api::error::MemoryError::NotFound(_)), - "{error:?}" - ); -} - -#[test] -fn a_path_escape_does_not_become_a_caller_mistake() { - // The security-relevant one: a sandbox escape must not be reclassified as a - // malformed argument. - let error = error_to_tinycortex(tinymemory_api::error::MemoryError::PathEscape( - "symlink leaves workspace".to_string(), - )); - assert!( - matches!(error, tinycortex_api::error::MemoryError::PathEscape(_)), - "{error:?}" - ); -} - -#[test] -fn an_unsupported_capability_keeps_its_family_name() { - let error = error_to_tinycortex(tinymemory_api::error::MemoryError::unsupported_raw( - "vendor_extension", - )); - match error { - tinycortex_api::error::MemoryError::Unsupported { capability } => { - assert_eq!(capability, "vendor_extension"); - } - other => panic!("expected Unsupported, got {other:?}"), - } -} - -#[test] -fn taint_and_category_do_not_go_through_serde() { - // Guards the deliberate exception. Taint decides whether externally-sourced - // content is treated as internal-trust content, so it uses the audited - // destructuring conversion in `tinymemory_tinycortex::convert` rather than a - // round trip that would tolerate an added variant. - use tinymemory_tinycortex::convert; - - for taint in [ - tinycortex_api::types::MemoryTaint::Internal, - tinycortex_api::types::MemoryTaint::ExternalSync, - ] { - let crossed = convert::taint_to_tinymemory(taint); - assert_eq!(convert::taint_to_tinycortex(crossed), taint); - } - - let category = tinycortex_api::types::MemoryCategory::Custom("notes".to_string()); - let crossed = convert::category_to_tinymemory(category.clone()); - assert_eq!(convert::category_to_tinycortex(crossed), category); -} diff --git a/src/openhuman/memory/guard/audit.rs b/src/openhuman/memory/guard/audit.rs index b4ef1f1cf6..11e2c50fa1 100644 --- a/src/openhuman/memory/guard/audit.rs +++ b/src/openhuman/memory/guard/audit.rs @@ -28,7 +28,7 @@ //! published. One bus event per memory read would flood every subscriber on the //! hot path for no operator benefit; a refusal is the rare, actionable event. -use tinycortex_api::capabilities::Capability; +use crate::openhuman::memory::api::capabilities::Capability; use crate::core::bus::BUS; use crate::core::events::DomainEvent; diff --git a/src/openhuman/memory/guard/budget.rs b/src/openhuman/memory/guard/budget.rs index 7a84e6581b..ce9b4c6687 100644 --- a/src/openhuman/memory/guard/budget.rs +++ b/src/openhuman/memory/guard/budget.rs @@ -12,7 +12,7 @@ //! on a byte index panics when that index is not a char boundary. Slicing at a //! char boundary is the only form that is both correct and total. -use tinycortex_api::types::MemoryEntry; +use crate::openhuman::memory::api::types::MemoryEntry; /// Truncate `content` to at most `max_chars` characters. /// diff --git a/src/openhuman/memory/guard/budget_tests.rs b/src/openhuman/memory/guard/budget_tests.rs index 3562523aac..0043a26d4d 100644 --- a/src/openhuman/memory/guard/budget_tests.rs +++ b/src/openhuman/memory/guard/budget_tests.rs @@ -1,7 +1,7 @@ //! Step 6 — the pure char-budget arithmetic. use super::*; -use tinycortex_api::types::{MemoryCategory, MemoryEntry, MemoryTaint}; +use crate::openhuman::memory::api::types::{MemoryCategory, MemoryEntry, MemoryTaint}; fn entry(content: &str) -> MemoryEntry { MemoryEntry { diff --git a/src/openhuman/memory/guard/families.rs b/src/openhuman/memory/guard/families.rs index e4717f9afa..e3630827ec 100644 --- a/src/openhuman/memory/guard/families.rs +++ b/src/openhuman/memory/guard/families.rs @@ -31,25 +31,25 @@ use std::sync::Arc; -use async_trait::async_trait; -use tinycortex_api::capabilities::Capability; -use tinycortex_api::chunks::Chunk; -use tinycortex_api::error::MemoryError; -use tinycortex_api::goals::GoalsDoc; -use tinycortex_api::provider::types::{ +use crate::openhuman::memory::api::capabilities::Capability; +use crate::openhuman::memory::api::chunks::Chunk; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::goals::GoalsDoc; +use crate::openhuman::memory::api::provider::types::{ DiffReport, EntityHit, IngestItem, IngestOutcome, MaintenanceReport, SnapshotRef, SourceItem, SourceScope, }; -use tinycortex_api::provider::{ +use crate::openhuman::memory::api::provider::{ MemoryDiff, MemoryDocuments, MemoryEntities, MemoryGoals, MemoryGraph, MemoryIngest, MemoryMaintenance, MemoryProvider, MemorySourceSink, MemoryToolMemory, MemoryTree, }; -use tinycortex_api::tool_memory::ToolMemoryRule; -use tinycortex_api::tree::{IngestRequest, QueryResult, TreeStatus}; -use tinycortex_api::types::{ +use crate::openhuman::memory::api::tool_memory::ToolMemoryRule; +use crate::openhuman::memory::api::tree::{IngestRequest, QueryResult, TreeStatus}; +use crate::openhuman::memory::api::types::{ GraphRelationRecord, MemoryKvRecord, MemoryTaint, NamespaceDocumentInput, NamespaceRetrievalContext, StoredMemoryDocument, }; +use async_trait::async_trait; use super::audit::{trace_allowed, NO_NAMESPACE}; use super::policy::GuardPolicy; @@ -241,6 +241,53 @@ impl MemoryDocuments for GuardedDocuments { self.family()?.get_document(namespace, key).await } + async fn list_documents( + &self, + namespace: Option<&str>, + ) -> Result { + self.policy.admit_read( + Capability::Documents, + "documents.list_documents", + namespace.unwrap_or(NO_NAMESPACE), + false, + )?; + self.family()?.list_documents(namespace).await + } + + async fn list_namespaces(&self) -> Result, MemoryError> { + self.policy.admit_read( + Capability::Documents, + "documents.list_namespaces", + NO_NAMESPACE, + false, + )?; + self.family()?.list_namespaces().await + } + + async fn delete_document( + &self, + namespace: &str, + document_id: &str, + ) -> Result { + self.policy.admit_write( + Capability::Documents, + "documents.delete_document", + namespace, + false, + )?; + self.family()?.delete_document(namespace, document_id).await + } + + async fn clear_namespace(&self, namespace: &str) -> Result<(), MemoryError> { + self.policy.admit_write( + Capability::Documents, + "documents.clear_namespace", + namespace, + false, + )?; + self.family()?.clear_namespace(namespace).await + } + async fn query_documents( &self, namespace: &str, @@ -259,6 +306,20 @@ impl MemoryDocuments for GuardedDocuments { .query_documents(namespace, &query, limit) .await } + + async fn recall_documents( + &self, + namespace: &str, + limit: usize, + ) -> Result { + self.policy.admit_read( + Capability::Documents, + "documents.recall_documents", + namespace, + false, + )?; + self.family()?.recall_documents(namespace, limit).await + } } // ── Tree ───────────────────────────────────────────────────────────────────── @@ -433,6 +494,16 @@ impl MemoryGraph for GuardedGraph { self.family()?.kv_put(namespace, key, value).await } + async fn kv_delete(&self, namespace: Option<&str>, key: &str) -> Result { + self.policy.admit_write( + Capability::Graph, + "graph.kv_delete", + graph_ns(namespace), + false, + )?; + self.family()?.kv_delete(namespace, key).await + } + async fn kv_list( &self, namespace: Option<&str>, diff --git a/src/openhuman/memory/guard/families_tests.rs b/src/openhuman/memory/guard/families_tests.rs index a75409a045..c30eae7d72 100644 --- a/src/openhuman/memory/guard/families_tests.rs +++ b/src/openhuman/memory/guard/families_tests.rs @@ -1,10 +1,10 @@ //! The wrapped-accessor property — the reason this milestone exists — plus //! step 2, which lives on `GuardedTree::query_source`. -use tinycortex_api::provider::types::SourceScope; -use tinycortex_api::provider::{MemoryProvider, MemoryTree}; -use tinycortex_api::tree::IngestRequest; -use tinycortex_api::types::MemoryTaint; +use crate::openhuman::memory::api::provider::types::SourceScope; +use crate::openhuman::memory::api::provider::{MemoryProvider, MemoryTree}; +use crate::openhuman::memory::api::tree::IngestRequest; +use crate::openhuman::memory::api::types::MemoryTaint; use crate::openhuman::memory::guard::test_support::{ document, embedded_policy, external_policy, guarded, diff --git a/src/openhuman/memory/guard/mandatory.rs b/src/openhuman/memory/guard/mandatory.rs index 195791a629..736ece3da0 100644 --- a/src/openhuman/memory/guard/mandatory.rs +++ b/src/openhuman/memory/guard/mandatory.rs @@ -1,13 +1,17 @@ //! The three mandatory families on [`MemoryGuard`] — where steps 3, 4 and 6 //! land for the always-present surface. +use crate::openhuman::memory::api::capabilities::Capability; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::provider::types::{ + ExportPage, ExportRecord, ImportOutcome, SourceScope, +}; +use crate::openhuman::memory::api::provider::{MemoryCore, MemoryPortability, MemoryRecall}; +use crate::openhuman::memory::api::recall::OwnedRecallOpts; +use crate::openhuman::memory::api::types::{ + MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, +}; use async_trait::async_trait; -use tinycortex_api::capabilities::Capability; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::{ExportPage, ExportRecord, ImportOutcome, SourceScope}; -use tinycortex_api::provider::{MemoryCore, MemoryPortability, MemoryRecall}; -use tinycortex_api::recall::OwnedRecallOpts; -use tinycortex_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; use super::audit::{trace_allowed, trace_budget, NO_NAMESPACE}; use super::budget::{truncate_content, truncate_entries}; diff --git a/src/openhuman/memory/guard/mod.rs b/src/openhuman/memory/guard/mod.rs index ff5b36e8da..1dfbca386e 100644 --- a/src/openhuman/memory/guard/mod.rs +++ b/src/openhuman/memory/guard/mod.rs @@ -3,7 +3,7 @@ //! //! ## The shape, and why it is this shape //! -//! The guard implements [`MemoryProvider`](tinycortex_api::provider::MemoryProvider) +//! The guard implements [`MemoryProvider`](crate::openhuman::memory::api::provider::MemoryProvider) //! over an `Arc`. That makes it *transparent* — a caller //! writes the same code against the guard as against the driver — and it makes //! the guard *unskippable by construction* for anyone holding it, because there diff --git a/src/openhuman/memory/guard/policy.rs b/src/openhuman/memory/guard/policy.rs index dfe66bbbed..e843dcde6d 100644 --- a/src/openhuman/memory/guard/policy.rs +++ b/src/openhuman/memory/guard/policy.rs @@ -35,10 +35,10 @@ use std::borrow::Cow; use std::sync::Arc; -use tinycortex_api::capabilities::Capability; -use tinycortex_api::error::MemoryError; -use tinycortex_api::provider::types::SourceScope; -use tinycortex_api::types::MemoryTaint; +use crate::openhuman::memory::api::capabilities::Capability; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::provider::types::SourceScope; +use crate::openhuman::memory::api::types::MemoryTaint; use crate::core::subsystem::DriverClass; use crate::openhuman::config::schema::MemoryHooksConfig; @@ -344,17 +344,6 @@ impl GuardPolicy { /// same time as the class check that selects it. pub fn redact_outbound<'a>(&self, content: &'a str) -> Cow<'a, str> { match self.class { - // `Module` sits with the in-process classes, and the test for this - // grouping is "does the content leave the device", not "is the code - // compiled in". A loaded module is in this address space and makes no - // egress of its own — the memory module's embeddings go back *to* the - // host, which is the whole point of that split — so there is no - // transfer here to disclose or scrub. - // - // Scrubbing would also be actively destructive for the same reason it - // would be for `Embedded`: these are the user's own memory writes, and - // sanitizing them would silently corrupt the data being stored rather - // than protect anything. DriverClass::Embedded | DriverClass::Module | DriverClass::Null => { Cow::Borrowed(content) } @@ -369,7 +358,6 @@ impl GuardPolicy { /// unmodified pass-through for embedded and null drivers. pub fn redact_outbound_json(&self, value: serde_json::Value) -> serde_json::Value { match self.class { - // Same grouping and the same reason as `redact_outbound`. DriverClass::Embedded | DriverClass::Module | DriverClass::Null => value, DriverClass::External => { crate::openhuman::memory::store::safety::sanitize_json(&value).value diff --git a/src/openhuman/memory/guard/provider.rs b/src/openhuman/memory/guard/provider.rs index 29353bcf61..84698bd727 100644 --- a/src/openhuman/memory/guard/provider.rs +++ b/src/openhuman/memory/guard/provider.rs @@ -2,14 +2,14 @@ use std::sync::Arc; -use async_trait::async_trait; -use tinycortex_api::capabilities::{Capabilities, Capability}; -use tinycortex_api::error::MemoryError; -use tinycortex_api::health::MemoryHealth; -use tinycortex_api::provider::{ +use crate::openhuman::memory::api::capabilities::{Capabilities, Capability}; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::health::MemoryHealth; +use crate::openhuman::memory::api::provider::{ MemoryDiff, MemoryDocuments, MemoryEntities, MemoryGoals, MemoryGraph, MemoryIngest, MemoryMaintenance, MemoryProvider, MemorySourceSink, MemoryToolMemory, MemoryTree, }; +use async_trait::async_trait; use super::families::{ GuardedDiff, GuardedDocuments, GuardedEntities, GuardedGoals, GuardedGraph, GuardedIngest, diff --git a/src/openhuman/memory/guard/provider_tests.rs b/src/openhuman/memory/guard/provider_tests.rs index bc0a6dddff..16852aa182 100644 --- a/src/openhuman/memory/guard/provider_tests.rs +++ b/src/openhuman/memory/guard/provider_tests.rs @@ -4,14 +4,14 @@ use super::*; use std::sync::Arc; -use tinycortex_api::capabilities::{Capabilities, Capability}; -use tinycortex_api::null::NullMemoryProvider; -use tinycortex_api::provider::types::SourceScope; -use tinycortex_api::provider::{ +use crate::openhuman::memory::api::capabilities::{Capabilities, Capability}; +use crate::openhuman::memory::api::null::NullMemoryProvider; +use crate::openhuman::memory::api::provider::types::SourceScope; +use crate::openhuman::memory::api::provider::{ audit_provider, MemoryCore, MemoryPortability, MemoryProvider, MemoryRecall, }; -use tinycortex_api::recall::OwnedRecallOpts; -use tinycortex_api::types::{MemoryCategory, MemoryTaint}; +use crate::openhuman::memory::api::recall::OwnedRecallOpts; +use crate::openhuman::memory::api::types::{MemoryCategory, MemoryTaint}; use crate::core::bus::BUS; use crate::core::events::DomainEvent; diff --git a/src/openhuman/memory/guard/test_support.rs b/src/openhuman/memory/guard/test_support.rs index 44c2eaa479..c285b747a5 100644 --- a/src/openhuman/memory/guard/test_support.rs +++ b/src/openhuman/memory/guard/test_support.rs @@ -11,28 +11,28 @@ use std::sync::{Arc, Mutex}; -use async_trait::async_trait; -use tinycortex_api::capabilities::Capabilities; -use tinycortex_api::chunks::Chunk; -use tinycortex_api::error::MemoryError; -use tinycortex_api::goals::GoalsDoc; -use tinycortex_api::health::MemoryHealth; -use tinycortex_api::provider::types::{ +use crate::openhuman::memory::api::capabilities::Capabilities; +use crate::openhuman::memory::api::chunks::Chunk; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::goals::GoalsDoc; +use crate::openhuman::memory::api::health::MemoryHealth; +use crate::openhuman::memory::api::provider::types::{ DiffReport, EntityHit, ExportPage, ExportRecord, ImportOutcome, IngestItem, IngestOutcome, MaintenanceReport, SnapshotRef, SourceItem, SourceScope, }; -use tinycortex_api::provider::{ +use crate::openhuman::memory::api::provider::{ MemoryCore, MemoryDiff, MemoryDocuments, MemoryEntities, MemoryGoals, MemoryGraph, MemoryIngest, MemoryMaintenance, MemoryPortability, MemoryProvider, MemoryRecall, MemorySourceSink, MemoryToolMemory, MemoryTree, }; -use tinycortex_api::recall::OwnedRecallOpts; -use tinycortex_api::tool_memory::ToolMemoryRule; -use tinycortex_api::tree::{IngestRequest, QueryResult, TreeStatus}; -use tinycortex_api::types::{ +use crate::openhuman::memory::api::recall::OwnedRecallOpts; +use crate::openhuman::memory::api::tool_memory::ToolMemoryRule; +use crate::openhuman::memory::api::tree::{IngestRequest, QueryResult, TreeStatus}; +use crate::openhuman::memory::api::types::{ GraphRelationRecord, MemoryCategory, MemoryEntry, MemoryKvRecord, MemoryTaint, NamespaceDocumentInput, NamespaceRetrievalContext, NamespaceSummary, StoredMemoryDocument, }; +use async_trait::async_trait; /// One call that reached the driver. #[derive(Debug, Clone, PartialEq)] @@ -338,6 +338,33 @@ impl MemoryDocuments for RecordingProvider { Ok(None) } + async fn list_documents( + &self, + _namespace: Option<&str>, + ) -> Result { + self.record(Call::plain("documents.list_documents")); + Ok(serde_json::json!({"documents": []})) + } + + async fn list_namespaces(&self) -> Result, MemoryError> { + self.record(Call::plain("documents.list_namespaces")); + Ok(vec![]) + } + + async fn delete_document( + &self, + _namespace: &str, + _document_id: &str, + ) -> Result { + self.record(Call::plain("documents.delete_document")); + Ok(serde_json::json!({"deleted": false})) + } + + async fn clear_namespace(&self, _namespace: &str) -> Result<(), MemoryError> { + self.record(Call::plain("documents.clear_namespace")); + Ok(()) + } + async fn query_documents( &self, namespace: &str, @@ -357,6 +384,20 @@ impl MemoryDocuments for RecordingProvider { hits: vec![], }) } + + async fn recall_documents( + &self, + namespace: &str, + _limit: usize, + ) -> Result { + self.record(Call::plain("documents.recall_documents")); + Ok(NamespaceRetrievalContext { + namespace: namespace.to_string(), + query: None, + context_text: String::new(), + hits: vec![], + }) + } } #[async_trait] @@ -471,6 +512,11 @@ impl MemoryGraph for RecordingProvider { Ok(()) } + async fn kv_delete(&self, _namespace: Option<&str>, _key: &str) -> Result { + self.record(Call::plain("graph.kv_delete")); + Ok(false) + } + async fn kv_list( &self, _namespace: Option<&str>, diff --git a/src/openhuman/memory/mod.rs b/src/openhuman/memory/mod.rs index 6278779b41..d176ac3c0a 100644 --- a/src/openhuman/memory/mod.rs +++ b/src/openhuman/memory/mod.rs @@ -24,6 +24,7 @@ //! resolving unchanged. Prefer `tinymemory_core::…` in new code. pub mod agent; +pub mod api; pub mod binding; pub mod driver; pub mod guard; diff --git a/src/openhuman/memory/ops/documents.rs b/src/openhuman/memory/ops/documents.rs index 29782b64b6..d30b48d168 100644 --- a/src/openhuman/memory/ops/documents.rs +++ b/src/openhuman/memory/ops/documents.rs @@ -5,17 +5,17 @@ use serde::{Deserialize, Serialize}; -use crate::core::subsystem::DriverClass; -use crate::openhuman::memory::store::{NamespaceDocumentInput, NamespaceRetrievalContext}; +use crate::openhuman::memory::api::provider::MemoryProvider; +use crate::openhuman::memory::api::types::NamespaceDocumentInput; +use crate::openhuman::memory::store::NamespaceRetrievalContext; use crate::openhuman::memory::{ ApiEnvelope, DeleteDocumentRequest, DeleteDocumentResponse, EmptyRequest, ListDocumentsRequest, - ListDocumentsResponse, ListNamespacesResponse, MemoryIngestionConfig, MemoryIngestionRequest, - MemoryIngestionResult, MemoryInitRequest, MemoryInitResponse, MemoryRecallItem, PaginationMeta, - QueryNamespaceRequest, QueryNamespaceResponse, RecallContextRequest, RecallContextResponse, - RecallMemoriesRequest, RecallMemoriesResponse, + ListDocumentsResponse, ListNamespacesResponse, MemoryIngestionConfig, MemoryIngestionResult, + MemoryInitRequest, MemoryInitResponse, MemoryRecallItem, PaginationMeta, QueryNamespaceRequest, + QueryNamespaceResponse, RecallContextRequest, RecallContextResponse, RecallMemoriesRequest, + RecallMemoriesResponse, }; use crate::rpc::RpcOutcome; -use tinycortex_api::provider::MemoryProvider; use super::envelope::{envelope, error_envelope, memory_counts}; use super::guard::active_memory_guard; @@ -165,8 +165,14 @@ pub struct PutDocResult { /// Lists all namespaces in the memory system. pub async fn namespace_list() -> Result>, String> { - let client = active_memory_client().await?; - let namespaces = client.list_namespaces().await?; + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let namespaces = documents + .list_namespaces() + .await + .map_err(|error| error.to_string())?; Ok(RpcOutcome::single_log( namespaces, "memory namespaces listed", @@ -209,7 +215,7 @@ pub async fn doc_put(params: PutDocParams) -> Result, S // RPC-driven doc puts come from the user / agent — Internal. // External-sync ingest paths bypass this RPC and call // `store_skill_sync` directly with their own taint label. - taint: crate::openhuman::memory::MemoryTaint::Internal, + taint: crate::openhuman::memory::api::types::MemoryTaint::Internal, }) .await .map_err(|e| e.to_string())?; @@ -223,26 +229,48 @@ pub async fn doc_put(params: PutDocParams) -> Result, S pub async fn doc_ingest( params: IngestDocParams, ) -> Result, String> { - let client = active_memory_client().await?; - let result = client - .ingest_doc(MemoryIngestionRequest { - document: NamespaceDocumentInput { - namespace: params.namespace, - key: params.key, - title: params.title, - content: params.content, - source_type: params.source_type, - priority: params.priority, - tags: params.tags, - metadata: params.metadata, - category: params.category, - session_id: params.session_id, - document_id: params.document_id, - taint: crate::openhuman::memory::MemoryTaint::Internal, - }, - config: params.config.unwrap_or_default(), + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let namespace = params.namespace; + let tags = params.tags; + let document_id = documents + .put_document(NamespaceDocumentInput { + namespace: namespace.clone(), + key: params.key, + title: params.title, + content: params.content, + source_type: params.source_type, + priority: params.priority, + tags: tags.clone(), + metadata: params.metadata, + category: params.category, + session_id: params.session_id, + document_id: params.document_id, + taint: crate::openhuman::memory::api::types::MemoryTaint::Internal, }) - .await?; + .await + .map_err(|error| error.to_string())?; + // Chunking and graph extraction are driver-owned behind `put_document`. + // The historical RPC response exposed the embedded engine's synchronous + // extraction details, which a module boundary cannot observe. Preserve the + // wire shape while reporting only the facts the host actually knows. + let _driver_owned_config = params.config; + let result = MemoryIngestionResult { + document_id, + namespace, + model_name: "driver-managed".to_string(), + extraction_mode: "driver-managed".to_string(), + chunk_count: 0, + entity_count: 0, + relation_count: 0, + preference_count: 0, + decision_count: 0, + tags, + entities: Vec::new(), + relations: Vec::new(), + }; let msg = format!( "ingested document — {} entities, {} relations, {} chunks", result.entity_count, result.relation_count, result.chunk_count, @@ -254,48 +282,27 @@ pub async fn doc_ingest( pub async fn doc_list( params: Option, ) -> Result, String> { - let client = active_memory_client().await?; - let docs = client - .list_documents(params.as_ref().map(|v| v.namespace.as_str())) - .await?; - Ok(RpcOutcome::single_log(docs, "memory documents listed")) -} - -/// Refuse an embedded-store-only operation when the bound driver is not the -/// embedded engine. -/// -/// `delete_document` / `clear_namespace` / `doc_delete` operate on the local -/// embedded SQLite store through `active_memory_client` and have **no contract -/// twin** — `MemoryDocuments` has no `delete_document` or `clear_namespace` -/// method — so they cannot be routed through the contract. Under a null or -/// fallback binding the operator asked for memory to be disabled, yet these -/// handlers would otherwise still reach the store boot initialised and delete -/// persisted rows. This is the RPC half of the CLI's legacy-client gate -/// (`core::cli_capability::legacy_client_verdict`): an embedded-only operation -/// is only valid when the bound driver actually is the embedded engine. The -/// check runs through the guarded binding (`active_memory_guard`), so the -/// verdict always reflects the driver that bound, never a global slot. -async fn ensure_embedded_driver(operation: &str) -> Result<(), String> { let guard = active_memory_guard().await?; - if guard.policy().class() == DriverClass::Embedded { - return Ok(()); - } - Err(format!( - "memory driver `{}` is not the embedded TinyCortex driver, so `{operation}` is \ - unavailable: it operates on the local embedded store directly, and this \ - configuration bound a different driver. Change `[subsystems.memory] driver` in \ - your config.", - guard.driver_id() - )) + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let docs = documents + .list_documents(params.as_ref().map(|value| value.namespace.as_str())) + .await + .map_err(|error| error.to_string())?; + Ok(RpcOutcome::single_log(docs, "memory documents listed")) } /// Deletes a document from a namespace. pub async fn doc_delete(params: DeleteDocParams) -> Result, String> { - ensure_embedded_driver("doc_delete").await?; - let client = active_memory_client().await?; - let result = client + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let result = documents .delete_document(¶ms.namespace, ¶ms.document_id) - .await?; + .await + .map_err(|error| error.to_string())?; Ok(RpcOutcome::single_log(result, "memory document deleted")) } @@ -303,10 +310,15 @@ pub async fn doc_delete(params: DeleteDocParams) -> Result Result, String> { - ensure_embedded_driver("clear_namespace").await?; - let client = active_memory_client().await?; + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; log::debug!("[memory] clear_namespace RPC invoked"); - client.clear_namespace(¶ms.namespace).await?; + documents + .clear_namespace(¶ms.namespace) + .await + .map_err(|error| error.to_string())?; let msg = "memory namespace cleared".to_string(); Ok(RpcOutcome::single_log( ClearNamespaceResult { @@ -319,22 +331,38 @@ pub async fn clear_namespace( /// Queries a namespace for contextual information based on a natural language string. pub async fn context_query(params: QueryNamespaceParams) -> Result, String> { - let client = active_memory_client().await?; - let result = client - .query_namespace(¶ms.namespace, ¶ms.query, params.limit.unwrap_or(10)) - .await?; - Ok(RpcOutcome::single_log(result, "memory context queried")) + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let result = documents + .query_documents( + ¶ms.namespace, + ¶ms.query, + params.limit.unwrap_or(10) as usize, + ) + .await + .map_err(|error| error.to_string())?; + Ok(RpcOutcome::single_log( + result.context_text, + "memory context queried", + )) } /// Recalls contextual information from a namespace without a specific query. pub async fn context_recall( params: RecallNamespaceParams, ) -> Result>, String> { - let client = active_memory_client().await?; - let result = client - .recall_namespace(¶ms.namespace, params.limit.unwrap_or(10)) - .await?; - Ok(RpcOutcome::single_log(result, "memory context recalled")) + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let result = documents + .recall_documents(¶ms.namespace, params.limit.unwrap_or(10) as usize) + .await + .map_err(|error| error.to_string())?; + let context = (!result.context_text.is_empty()).then_some(result.context_text); + Ok(RpcOutcome::single_log(context, "memory context recalled")) } // --------------------------------------------------------------------------- @@ -368,8 +396,14 @@ pub async fn memory_init( pub async fn memory_list_documents( request: ListDocumentsRequest, ) -> Result>, String> { - let client = active_memory_client().await?; - let raw = client.list_documents(request.namespace.as_deref()).await?; + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let raw = documents + .list_documents(request.namespace.as_deref()) + .await + .map_err(|error| error.to_string())?; let documents = parse_memory_document_summaries(raw)?; let count = documents.len(); Ok(envelope( @@ -391,8 +425,14 @@ pub async fn memory_list_documents( pub async fn memory_list_namespaces( _request: EmptyRequest, ) -> Result>, String> { - let client = active_memory_client().await?; - let namespaces = client.list_namespaces().await?; + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let namespaces = documents + .list_namespaces() + .await + .map_err(|error| error.to_string())?; let count = namespaces.len(); Ok(envelope( ListNamespacesResponse { namespaces, count }, @@ -405,11 +445,14 @@ pub async fn memory_list_namespaces( pub async fn memory_delete_document( request: DeleteDocumentRequest, ) -> Result>, String> { - ensure_embedded_driver("delete_document").await?; - let client = active_memory_client().await?; - let raw = client + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let raw = documents .delete_document(&request.namespace, &request.document_id) - .await?; + .await + .map_err(|error| error.to_string())?; let parsed: RawDeleteDocumentResult = serde_json::from_value(raw).map_err(|e| format!("decode delete document result: {e}"))?; Ok(envelope( @@ -809,15 +852,15 @@ mod tests { } /// Same store property as `kv_set_through_the_guard_…`: the guarded - /// `doc_put` must be readable by the unguarded client, not merely by the - /// sibling handler. + /// `doc_put` must be readable through the module-backed memory API, not + /// merely by the sibling handler. /// /// The taint half of this re-point is not asserted here because no read /// path in `MemoryClient` projects the stored taint column back out. /// `GuardPolicy::stamp_taint`'s monotone-raise behaviour is pinned in /// `memory::guard::policy_tests` instead. #[tokio::test] - async fn doc_put_through_the_guard_is_visible_to_the_unguarded_client() { + async fn doc_put_through_the_guard_is_visible_to_the_memory_api() { let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK .lock() .await; @@ -838,27 +881,23 @@ mod tests { .expect("guarded doc_put"); assert!(!put.value.document_id.is_empty()); - let client = active_memory_client().await.expect("client"); - let raw = client + let guard = active_memory_guard().await.expect("guard"); + let documents = guard.inner().as_documents().expect("documents family"); + let raw = documents .list_documents(Some(namespace.as_str())) .await - .expect("unguarded list_documents"); + .expect("module-backed list_documents"); let docs = raw .get("documents") .and_then(|v| v.as_array()) .expect("documents array"); assert!( docs.iter().any(|doc| doc["key"] == key), - "the unguarded client must see the guarded write" + "the module-backed memory API must see the guarded write" ); } - /// Pins the null-binding refusal for the destructive embedded-only ops - /// (the `all.rs` registration thread). Under `driver = "null"` the operator - /// asked for memory to be disabled, yet `clear_namespace` / - /// `delete_document` would still reach the embedded store boot initialised - /// and delete persisted rows. They must refuse with a config-fact message, - /// exactly as the CLI's legacy-client gate does. + /// Pins the null-binding refusal for destructive document operations. #[tokio::test] async fn destructive_ops_refuse_when_bound_driver_is_null() { let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK @@ -882,7 +921,7 @@ mod tests { .await .expect_err("clear_namespace must refuse under a null binding"); assert!( - err.contains("not the embedded TinyCortex driver"), + err.contains("does not support the documents family"), "refusal must explain the binding: {err}" ); @@ -893,7 +932,7 @@ mod tests { .await .expect_err("doc_delete must refuse under a null binding"); assert!( - err.contains("not the embedded TinyCortex driver"), + err.contains("does not support the documents family"), "refusal must explain the binding: {err}" ); @@ -904,7 +943,7 @@ mod tests { .await .expect_err("memory_delete_document must refuse under a null binding"); assert!( - err.contains("not the embedded TinyCortex driver"), + err.contains("does not support the documents family"), "refusal must explain the binding: {err}" ); }) diff --git a/src/openhuman/memory/ops/guard_tests.rs b/src/openhuman/memory/ops/guard_tests.rs index c655b980f8..f8f3380b03 100644 --- a/src/openhuman/memory/ops/guard_tests.rs +++ b/src/openhuman/memory/ops/guard_tests.rs @@ -39,8 +39,8 @@ async fn falls_back_to_the_globally_bound_workspace_when_there_is_no_context() { /// handler routed through it still reports the embedded driver in status and /// spans. #[tokio::test] -async fn guards_the_embedded_driver_and_keeps_its_identity() { - use tinycortex_api::provider::MemoryProvider; +async fn guards_the_module_driver_and_keeps_its_identity() { + use crate::openhuman::memory::api::provider::MemoryProvider; let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK .lock() @@ -50,7 +50,7 @@ async fn guards_the_embedded_driver_and_keeps_its_identity() { let guard = active_memory_guard().await.expect("guard resolves"); assert_eq!( guard.driver_id(), - crate::openhuman::memory::driver::embedded::EMBEDDED_DRIVER_ID + crate::openhuman::memory::binding::MODULE_ID ); assert!(guard.as_documents().is_some()); assert!(guard.as_graph().is_some()); diff --git a/src/openhuman/memory/ops/kv_graph.rs b/src/openhuman/memory/ops/kv_graph.rs index 8cb9bad3bc..1b79c41f4d 100644 --- a/src/openhuman/memory/ops/kv_graph.rs +++ b/src/openhuman/memory/ops/kv_graph.rs @@ -2,12 +2,11 @@ use serde::Deserialize; -use tinycortex_api::provider::MemoryProvider; +use crate::openhuman::memory::api::provider::MemoryProvider; use crate::rpc::RpcOutcome; use super::guard::active_memory_guard; -use super::helpers::active_memory_client; /// Parameters for the `kv_set` RPC method. #[derive(Debug, Deserialize)] @@ -69,13 +68,8 @@ pub struct GraphQueryParams { /// Sets a key-value pair in the memory store. /// /// Routed through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard) -/// rather than the bare client. `MemoryGraph::kv_put` on the embedded driver is -/// `client.kv_set(namespace, key, &value)` — the same call on the same store, -/// so the only differences are the policy steps the guard adds (tier check, -/// audit span) and an error string that gains a `"kv_put: "` prefix. -/// -/// Its three siblings in this file deliberately stay on the bare client; see -/// `docs/specs/memory-guard-allowlist.md`. +/// and the shared [`MemoryGraph`](crate::openhuman::memory::api::provider::MemoryGraph) +/// API, as are the other KV and graph handlers in this file. pub async fn kv_set(params: KvSetParams) -> Result, String> { let guard = active_memory_guard().await?; let graph = guard @@ -92,19 +86,28 @@ pub async fn kv_set(params: KvSetParams) -> Result, String> { pub async fn kv_get( params: KvGetDeleteParams, ) -> Result>, String> { - let client = active_memory_client().await?; - let value = client + let guard = active_memory_guard().await?; + let graph = guard + .as_graph() + .ok_or_else(|| "memory driver does not support the graph family".to_string())?; + let value = graph .kv_get(params.namespace.as_deref(), ¶ms.key) - .await?; + .await + .map_err(|error| error.to_string())? + .map(|record| record.value); Ok(RpcOutcome::single_log(value, "memory kv get")) } /// Deletes a key-value pair from the memory store. pub async fn kv_delete(params: KvGetDeleteParams) -> Result, String> { - let client = active_memory_client().await?; - let deleted = client + let guard = active_memory_guard().await?; + let graph = guard + .as_graph() + .ok_or_else(|| "memory driver does not support the graph family".to_string())?; + let deleted = graph .kv_delete(params.namespace.as_deref(), ¶ms.key) - .await?; + .await + .map_err(|error| error.to_string())?; Ok(RpcOutcome::single_log(deleted, "memory kv delete")) } @@ -112,8 +115,17 @@ pub async fn kv_delete(params: KvGetDeleteParams) -> Result, St pub async fn kv_list_namespace( params: super::documents::NamespaceOnlyParams, ) -> Result>, String> { - let client = active_memory_client().await?; - let rows = client.kv_list_namespace(¶ms.namespace).await?; + let guard = active_memory_guard().await?; + let graph = guard + .as_graph() + .ok_or_else(|| "memory driver does not support the graph family".to_string())?; + let rows = graph + .kv_list(Some(¶ms.namespace), None, usize::MAX) + .await + .map_err(|error| error.to_string())? + .into_iter() + .map(|record| serde_json::to_value(record).map_err(|error| error.to_string())) + .collect::, _>>()?; Ok(RpcOutcome::single_log(rows, "memory namespace kv listed")) } @@ -123,16 +135,25 @@ pub async fn kv_list_namespace( /// Upserts a relation triple in the knowledge graph. pub async fn graph_upsert(params: GraphUpsertParams) -> Result, String> { - let client = active_memory_client().await?; - client - .graph_upsert( - params.namespace.as_deref(), - ¶ms.subject, - ¶ms.predicate, - ¶ms.object, - ¶ms.attrs, - ) - .await?; + let guard = active_memory_guard().await?; + let graph = guard + .as_graph() + .ok_or_else(|| "memory driver does not support the graph family".to_string())?; + graph + .put_relation(crate::openhuman::memory::api::types::GraphRelationRecord { + namespace: params.namespace, + subject: params.subject, + predicate: params.predicate, + object: params.object, + attrs: params.attrs, + updated_at: 0.0, + evidence_count: 1, + order_index: None, + document_ids: vec![], + chunk_ids: vec![], + }) + .await + .map_err(|error| error.to_string())?; Ok(RpcOutcome::single_log(true, "memory graph upserted")) } @@ -140,14 +161,22 @@ pub async fn graph_upsert(params: GraphUpsertParams) -> Result, pub async fn graph_query( params: GraphQueryParams, ) -> Result>, String> { - let client = active_memory_client().await?; - let rows = client - .graph_query( + let guard = active_memory_guard().await?; + let graph = guard + .as_graph() + .ok_or_else(|| "memory driver does not support the graph family".to_string())?; + let rows = graph + .relations( params.namespace.as_deref(), params.subject.as_deref(), params.predicate.as_deref(), + usize::MAX, ) - .await?; + .await + .map_err(|error| error.to_string())? + .into_iter() + .map(|record| serde_json::to_value(record).map_err(|error| error.to_string())) + .collect::, _>>()?; Ok(RpcOutcome::single_log(rows, "memory graph queried")) } @@ -258,14 +287,12 @@ mod tests { assert_eq!(queried.value[0]["object"], "ATLAS"); } - /// The guarded `kv_set` must land in the **same** store the unguarded - /// readers use. This is the failure a re-point can hide: routing through a - /// binding over a different workspace still returns `Ok`, it just writes - /// somewhere nobody reads. Asserted against the raw client rather than the - /// sibling handler so a shared bug in `active_memory_client` cannot make - /// both halves agree while both are wrong. + /// The guarded `kv_set` must land in the **same** module-backed provider + /// returned by the shared memory API. This is the failure a re-point can + /// hide: routing through a binding over a different workspace still + /// returns `Ok`, it just writes somewhere nobody reads. #[tokio::test] - async fn kv_set_through_the_guard_is_visible_to_the_unguarded_client() { + async fn kv_set_through_the_guard_is_visible_to_the_memory_api() { let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK .lock() .await; @@ -284,11 +311,15 @@ mod tests { .await .expect("guarded kv set"); - let client = active_memory_client().await.expect("client"); - let raw = client + let guard = active_memory_guard().await.expect("guard"); + let graph = guard.inner().as_graph().expect("graph family"); + let raw = graph .kv_get(Some(namespace.as_str()), &key) .await - .expect("unguarded kv get"); - assert_eq!(raw, Some(serde_json::json!({"via": "guard"}))); + .expect("module-backed kv get"); + assert_eq!( + raw.map(|record| record.value), + Some(serde_json::json!({"via": "guard"})) + ); } } diff --git a/src/openhuman/memory/ops/learn.rs b/src/openhuman/memory/ops/learn.rs index 471d4762c6..d0dd73305e 100644 --- a/src/openhuman/memory/ops/learn.rs +++ b/src/openhuman/memory/ops/learn.rs @@ -2,9 +2,10 @@ use std::collections::BTreeSet; +use crate::openhuman::memory::api::provider::MemoryProvider; use crate::rpc::RpcOutcome; -use super::helpers::active_memory_client; +use super::guard::active_memory_guard; /// Per-namespace outcome for `memory_learn_all`. #[derive(Debug, serde::Serialize)] @@ -45,8 +46,14 @@ pub async fn memory_learn_all( ); // Resolve the target namespace list. - let client = active_memory_client().await?; - let all_ns = client.list_namespaces().await?; + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let all_ns = documents + .list_namespaces() + .await + .map_err(|error| error.to_string())?; tracing::debug!("[memory.learn] available namespaces: {:?}", all_ns); let target_ns: Vec = match ¶ms.namespaces { diff --git a/src/openhuman/memory/ops/provider.rs b/src/openhuman/memory/ops/provider.rs index 597d3261a2..15f190d722 100644 --- a/src/openhuman/memory/ops/provider.rs +++ b/src/openhuman/memory/ops/provider.rs @@ -113,7 +113,7 @@ fn unresolved_status(reason: String) -> SubsystemStatus { health: DriverHealth::down(reason.clone()).as_str().to_string(), health_reason: Some(reason.clone()), contract_version: crate::core::subsystem::format_contract_version( - tinycortex_api::CONTRACT_VERSION, + crate::openhuman::memory::api::CONTRACT_VERSION, ), capabilities: Vec::new(), fell_back_from: None, @@ -142,7 +142,9 @@ mod tests { // build fact, independent of whether anything bound. assert_eq!( status.contract_version, - crate::core::subsystem::format_contract_version(tinycortex_api::CONTRACT_VERSION) + crate::core::subsystem::format_contract_version( + crate::openhuman::memory::api::CONTRACT_VERSION + ) ); } @@ -155,14 +157,16 @@ mod tests { let status = status_from_binding(&binding).await; assert_eq!(status.slot, "memory"); - // The default `[subsystems.memory] driver` is the embedded tinycortex - // driver. - assert_eq!(status.driver, cfg.driver); - assert_eq!(status.class, "embedded"); + // The legacy default id is normalized to the compiled TinyMemory + // module at the binding boundary. + assert_eq!(status.driver, crate::openhuman::memory::binding::MODULE_ID); + assert_eq!(status.class, "module"); assert_eq!(status.health, "ready"); assert_eq!( status.contract_version, - crate::core::subsystem::format_contract_version(tinycortex_api::CONTRACT_VERSION) + crate::core::subsystem::format_contract_version( + crate::openhuman::memory::api::CONTRACT_VERSION + ) ); // All thirteen families, as of M3d. Spelled out rather than derived // from `Capabilities::all()` on purpose: this is the wire surface the diff --git a/src/openhuman/memory/ops/tool_memory.rs b/src/openhuman/memory/ops/tool_memory.rs index ea600a8510..ed63943f58 100644 --- a/src/openhuman/memory/ops/tool_memory.rs +++ b/src/openhuman/memory/ops/tool_memory.rs @@ -5,27 +5,22 @@ //! RPCs use, and the namespace they touch is exactly `tool-{tool_name}` — //! never `global` or `tool_effectiveness`. //! -//! Two of them — [`tool_rule_list`] and [`tool_rule_delete`] — reach it through -//! [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard) because their -//! contract twins are literal delegations to the same store. The other four -//! stay on [`open_store`]: `tool_rule_put` returns the *stored* rule (with -//! `created_at` preserved and `updated_at` refreshed) while the contract's -//! `put_tool_rule` returns unit, and `get_rule` / `list_rules_json` / -//! `rules_for_prompt` have no contract equivalent at all. See -//! `docs/specs/memory-guard-allowlist.md`. +//! Every handler reaches the module-backed [`MemoryToolMemory`](crate::openhuman::memory::api::provider::MemoryToolMemory) +//! API through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard), +//! including the host-shaped operations that compose multiple API methods to +//! preserve their historical response values. use serde::Deserialize; use serde_json::Value; +use std::sync::Arc; -use tinycortex_api::provider::MemoryProvider; +use crate::openhuman::memory::api::provider::MemoryProvider; -use crate::openhuman::memory::tool_memory::{ - tool_memory_store, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource, ToolMemoryStore, +use crate::openhuman::memory::api::tool_memory::{ + ToolMemoryPriority, ToolMemoryRule, ToolMemorySource, }; use crate::rpc::RpcOutcome; -use super::helpers::active_memory_client; - /// Parameters for `memory_tool_rule_put`. #[derive(Debug, Deserialize)] pub struct ToolRulePutParams { @@ -70,9 +65,8 @@ pub struct ToolRulesForPromptParams { pub tools: Vec, } -async fn open_store() -> Result { - let client = active_memory_client().await?; - Ok(tool_memory_store(client.memory_handle())) +async fn tool_memory_guard() -> Result, String> { + super::guard::active_memory_guard().await } /// Upsert a tool-scoped memory rule. @@ -80,7 +74,6 @@ pub async fn tool_rule_put( params: ToolRulePutParams, ) -> Result, String> { log::debug!("[tool-memory] rpc tool_rule_put tool={}", params.tool_name); - let store = open_store().await?; let mut rule = ToolMemoryRule::new( ¶ms.tool_name, ¶ms.rule, @@ -93,8 +86,14 @@ pub async fn tool_rule_put( rule.id = id; } } - let stored = store.put_rule(rule).await?; - Ok(RpcOutcome::single_log(stored, "tool memory rule stored")) + let guard = tool_memory_guard().await?; + guard + .as_tool_memory() + .ok_or_else(|| NO_TOOL_MEMORY.to_string())? + .put_tool_rule(rule.clone()) + .await + .map_err(|e| e.to_string())?; + Ok(RpcOutcome::single_log(rule, "tool memory rule stored")) } /// Fetch a tool-scoped rule by id. @@ -106,8 +105,15 @@ pub async fn tool_rule_get( params.tool_name, params.id ); - let store = open_store().await?; - let rule = store.get_rule(¶ms.tool_name, ¶ms.id).await?; + let guard = tool_memory_guard().await?; + let rule = guard + .as_tool_memory() + .ok_or_else(|| NO_TOOL_MEMORY.to_string())? + .tool_rules(¶ms.tool_name) + .await + .map_err(|e| e.to_string())? + .into_iter() + .find(|rule| rule.id == params.id); Ok(RpcOutcome::single_log(rule, "tool memory rule fetched")) } @@ -124,11 +130,9 @@ pub(crate) const NO_TOOL_MEMORY: &str = "memory driver does not support the tool /// List every tool-scoped rule for a tool. /// /// Routed through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard). -/// `MemoryToolMemory::tool_rules` on the embedded driver is -/// `tool_memory_store(self.memory()).list_rules(tool_name)` — the same store -/// over the same `Arc` [`open_store`] builds. The wire type matches -/// by identity, not conversion: `memory::tool_memory::ToolMemoryRule` **is** -/// `tinycortex_api::tool_memory::ToolMemoryRule`. +/// The wire type matches by identity, not conversion: +/// `memory::tool_memory::ToolMemoryRule` **is** +/// `crate::openhuman::memory::api::tool_memory::ToolMemoryRule`. pub async fn tool_rule_list( params: ToolRuleListParams, ) -> Result>, String> { @@ -145,9 +149,8 @@ pub async fn tool_rule_list( /// Delete a tool-scoped rule by id. /// -/// Routed through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard); -/// `MemoryToolMemory::delete_tool_rule` delegates to the same -/// `ToolMemoryStore::delete_rule` [`open_store`] would reach. +/// Routed through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard) +/// and the shared tool-memory API. pub async fn tool_rule_delete(params: ToolRuleRefParams) -> Result, String> { log::debug!( "[tool-memory] rpc tool_rule_delete tool={} id={}", @@ -183,9 +186,21 @@ pub async fn tool_rules_for_prompt( "[tool-memory] rpc tool_rules_for_prompt tools={:?}", params.tools ); - let store = open_store().await?; - let grouped = store.rules_for_prompt(¶ms.tools).await?; - let mut flat: Vec = grouped.into_values().flatten().collect(); + let guard = tool_memory_guard().await?; + let family = guard + .as_tool_memory() + .ok_or_else(|| NO_TOOL_MEMORY.to_string())?; + let mut flat = Vec::new(); + for tool in ¶ms.tools { + flat.extend( + family + .tool_rules(tool) + .await + .map_err(|e| e.to_string())? + .into_iter() + .filter(|rule| rule.priority.is_eager()), + ); + } flat.sort_by(|a, b| { b.priority .cmp(&a.priority) @@ -209,26 +224,31 @@ pub async fn tool_rules_json(params: ToolRuleListParams) -> Result String { - static NEXT_TOOL_ID: AtomicUsize = AtomicUsize::new(1); - let id = NEXT_TOOL_ID.fetch_add(1, Ordering::Relaxed); - format!("toolmem_test_{id}") + format!( + "toolmem_test_{}", + &uuid::Uuid::new_v4().as_simple().to_string()[..12] + ) } #[tokio::test] @@ -255,7 +275,7 @@ mod tests { assert_eq!(stored.priority, ToolMemoryPriority::Normal); assert_eq!( stored.source, - crate::openhuman::memory::tool_memory::ToolMemorySource::Programmatic + crate::openhuman::memory::api::tool_memory::ToolMemorySource::Programmatic ); assert_eq!(stored.tags, vec!["safety".to_string()]); assert!( @@ -369,13 +389,10 @@ mod tests { .await; } - /// The two guarded handlers and the four unguarded ones share one store. - /// `tool_rule_put` writes through `open_store()` (the bare client); - /// `tool_rule_list` and `tool_rule_delete` read and write through the - /// guard; `open_store()` is then asked directly whether the delete - /// actually happened. + /// Host-shaped put and guarded list/delete compose the same module-backed + /// tool-memory API. #[tokio::test] - async fn guarded_list_and_delete_share_the_store_with_the_unguarded_put() { + async fn guarded_list_and_delete_share_the_store_with_host_shaped_put() { let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK .lock() .await; @@ -391,7 +408,7 @@ mod tests { id: None, }) .await - .expect("unguarded put") + .expect("host-shaped put") .value; let listed = tool_rule_list(ToolRuleListParams { @@ -400,7 +417,7 @@ mod tests { .await .expect("guarded list") .value; - assert_eq!(listed.len(), 1, "the guard must see the unguarded write"); + assert_eq!(listed.len(), 1, "the guard must see the API write"); assert_eq!(listed[0].id, stored.id); let deleted = tool_rule_delete(ToolRuleRefParams { @@ -412,15 +429,13 @@ mod tests { .value; assert!(deleted); - let remaining = open_store() - .await - .expect("unguarded store") - .list_rules(&tool_name) + let remaining = tool_rule_list(ToolRuleListParams { tool_name }) .await - .expect("unguarded list"); + .expect("module-backed list") + .value; assert!( remaining.is_empty(), - "the unguarded store must observe the guarded delete" + "the module-backed provider must observe the guarded delete" ); } } diff --git a/src/openhuman/memory/schemas/documents.rs b/src/openhuman/memory/schemas/documents.rs index 415b249ff1..d868bf18b0 100644 --- a/src/openhuman/memory/schemas/documents.rs +++ b/src/openhuman/memory/schemas/documents.rs @@ -21,7 +21,7 @@ use super::{parse_params, to_json}; // --------------------------------------------------------------------------- // // This file is ONE RPC family by directory layout but THREE capability families -// by contract (`tinycortex_api::capabilities::Capability`), so M5.2 partitions +// by contract (`crate::openhuman::memory::api::capabilities::Capability`), so M5.2 partitions // it rather than tagging the whole file with a single capability: // // * core/recall — `Capability::Core` + `Capability::Recall`, both MANDATORY. diff --git a/src/openhuman/memory/tool_memory/capture.rs b/src/openhuman/memory/tool_memory/capture.rs index 36d37d5b48..fa08522c5d 100644 --- a/src/openhuman/memory/tool_memory/capture.rs +++ b/src/openhuman/memory/tool_memory/capture.rs @@ -33,9 +33,10 @@ use std::sync::Arc; use async_trait::async_trait; -use super::{tool_memory_store, ToolMemoryPriority, ToolMemorySource, ToolMemoryStore}; +use super::{tool_memory_store, ToolMemoryStore}; use crate::openhuman::agent::hooks::{PostTurnHook, ToolCallRecord, TurnContext}; use crate::openhuman::memory::Memory; +use tinycortex::memory::tool_memory::{ToolMemoryPriority, ToolMemorySource}; /// Maximum length (chars) of the captured rule body — keeps malformed or /// runaway input from bloating the namespace. @@ -460,7 +461,9 @@ mod tests { let mut flat: Vec<_> = prompt.into_values().flatten().collect(); flat.sort_by(|a, b| b.priority.cmp(&a.priority)); let rendered = - crate::openhuman::memory::tool_memory::prompt::render_tool_memory_rules(&flat); + crate::openhuman::memory::tool_memory::prompt::ToolMemoryRulesSection::new(flat) + .rendered() + .to_string(); assert!(rendered.contains("Never email Sarah")); assert!(rendered.contains("**[critical]**")); } diff --git a/src/openhuman/memory/tool_memory/prompt.rs b/src/openhuman/memory/tool_memory/prompt.rs index dee4d07b28..34f2065c2c 100644 --- a/src/openhuman/memory/tool_memory/prompt.rs +++ b/src/openhuman/memory/tool_memory/prompt.rs @@ -24,9 +24,75 @@ use anyhow::Result; use crate::openhuman::agent::context::prompt::{PromptContext, PromptSection}; -pub use tinycortex::memory::tool_memory::render::{ - render_tool_memory_rules, ToolMemoryRulesSection, TOOL_MEMORY_HEADING, -}; +use crate::openhuman::memory::api::tool_memory::{ToolMemoryPriority, ToolMemoryRule}; + +pub const TOOL_MEMORY_HEADING: &str = "## Tool-scoped rules"; +pub struct ToolMemoryRulesSection { + rendered: String, +} +impl ToolMemoryRulesSection { + pub fn new(rules: Vec) -> Self { + let rules: Vec = rules + .into_iter() + .filter_map(|rule| { + serde_json::to_value(rule) + .ok() + .and_then(|value| serde_json::from_value(value).ok()) + }) + .collect(); + Self { + rendered: render_tool_memory_rules(&rules), + } + } + pub fn empty() -> Self { + Self { + rendered: String::new(), + } + } + pub fn is_empty(&self) -> bool { + self.rendered.trim().is_empty() + } + pub fn rendered(&self) -> &str { + &self.rendered + } +} +pub fn render_tool_memory_rules(rules: &[ToolMemoryRule]) -> String { + if rules.is_empty() { + return String::new(); + } + let mut sorted: Vec<_> = rules.iter().collect(); + sorted.sort_by(|a, b| { + a.tool_name + .cmp(&b.tool_name) + .then_with(|| b.priority.cmp(&a.priority)) + .then_with(|| a.rule.cmp(&b.rule)) + .then_with(|| a.id.cmp(&b.id)) + }); + let mut out = format!("{TOOL_MEMORY_HEADING}\n\nThese rules are pinned by the user or by the safety pipeline. Treat every entry as a hard constraint when considering the matching tool — do not override them silently. Lower-priority guidance lives in the `tool-{{name}}` memory namespace and can be queried via `memory_recall` if needed.\n\n"); + let mut current = None; + for rule in sorted { + if current != Some(rule.tool_name.as_str()) { + if current.is_some() { + out.push('\n'); + } + out.push_str(&format!( + "### `{}`\n", + prompt_line(&rule.tool_name).replace('`', "'") + )); + current = Some(rule.tool_name.as_str()); + } + let priority = match rule.priority { + ToolMemoryPriority::Critical => "**[critical]**", + ToolMemoryPriority::High => "**[high]**", + ToolMemoryPriority::Normal => "**[normal]**", + }; + out.push_str(&format!("- {priority} {}\n", prompt_line(&rule.rule))); + } + out +} +fn prompt_line(value: &str) -> String { + value.split_whitespace().collect::>().join(" ") +} impl PromptSection for ToolMemoryRulesSection { fn name(&self) -> &str { diff --git a/src/openhuman/memory/tools/tool_memory/list.rs b/src/openhuman/memory/tools/tool_memory/list.rs index 665c2378ce..fab1359ffc 100644 --- a/src/openhuman/memory/tools/tool_memory/list.rs +++ b/src/openhuman/memory/tools/tool_memory/list.rs @@ -5,14 +5,14 @@ //! embedded driver is literally `tool_memory_store(self.memory()).list_rules(…)`, //! and the wire type matches by identity, not conversion: //! `memory::tool_memory::ToolMemoryRule` **is** -//! `tinycortex_api::tool_memory::ToolMemoryRule`. So the re-point is exact — +//! `crate::openhuman::memory::api::tool_memory::ToolMemoryRule`. So the re-point is exact — //! same rules, same order, same serialization — with `Capability::ToolMemory` //! admitted first. +use crate::openhuman::memory::api::provider::MemoryProvider; use async_trait::async_trait; use serde::Deserialize; use serde_json::json; -use tinycortex_api::provider::MemoryProvider; use crate::openhuman::memory::ops::guard::active_memory_guard; use crate::openhuman::memory::ops::tool_memory::NO_TOOL_MEMORY; diff --git a/src/openhuman/memory/tools/tool_memory/put.rs b/src/openhuman/memory/tools/tool_memory/put.rs index 1b09a1d713..2635c13926 100644 --- a/src/openhuman/memory/tools/tool_memory/put.rs +++ b/src/openhuman/memory/tools/tool_memory/put.rs @@ -21,15 +21,17 @@ //! store-level validation errors arrive as `MemoryError::Invalid` rather than as //! a raw string. +use crate::openhuman::memory::api::provider::MemoryProvider; use async_trait::async_trait; use serde::Deserialize; use serde_json::json; -use tinycortex_api::provider::MemoryProvider; +use crate::openhuman::memory::api::tool_memory::{ + ToolMemoryPriority, ToolMemoryRule, ToolMemorySource, +}; use crate::openhuman::memory::ops::guard::active_memory_guard; use crate::openhuman::memory::ops::tool_memory::NO_TOOL_MEMORY; use crate::openhuman::tools::traits::{Tool, ToolResult}; -use tinymemory_core::tool_memory::{ToolMemoryPriority, ToolMemoryRule, ToolMemorySource}; pub struct MemoryToolsPutTool; diff --git a/src/openhuman/modules/boot.rs b/src/openhuman/modules/boot.rs index 70768646af..352fc9907a 100644 --- a/src/openhuman/modules/boot.rs +++ b/src/openhuman/modules/boot.rs @@ -24,6 +24,7 @@ use crate::openhuman::config::Config; /// unavailable, and the feature says so at the point of use. Taking the core /// down because an optional codec is missing would be a worse trade. pub async fn load_declared_modules(config: &Config) { + super::memory::set_modules_policy(std::sync::Arc::new(config.clone())); if !config.modules.enabled { log::debug!("[modules] boot load skipped: modules are disabled in configuration"); return; @@ -102,28 +103,20 @@ mod tests { use crate::openhuman::config::Config; #[test] - fn tinymemory_is_not_eager_when_the_memory_driver_is_embedded() { - // The default config binds the embedded driver, so the module-backed - // TinyMemory record must not be treated as eager — otherwise every - // host with `modules.enabled` would pay a boot-time download for a - // driver it never binds. - let config = Config::default(); + fn tinymemory_is_not_eager_when_memory_is_disabled() { + let mut config = Config::default(); + config.subsystems.memory.driver = "null".to_string(); let record = super::registry::find(super::super::memory::MODULE_ID) .expect("tinymemory is a registered module"); assert!(!should_eager_load(record, &config)); } #[test] - fn tinymemory_is_eager_when_the_memory_driver_is_module_backed() { + fn tinymemory_is_eager_for_the_default_module_driver() { let mut config = Config::default(); - config.subsystems.memory.driver = "tinymemory".to_string(); - config.subsystems.memory.drivers.insert( - "tinymemory".to_string(), - tinymemory_api::host::MemoryDriverConfig { - class: Some("module".to_string()), - ..Default::default() - }, - ); + // The legacy persisted id aliases to the TinyMemory module until the + // shared API changes its default string. + config.subsystems.memory.driver = "tinycortex".to_string(); let record = super::registry::find(super::super::memory::MODULE_ID) .expect("tinymemory is a registered module"); assert!(should_eager_load(record, &config)); diff --git a/src/openhuman/modules/documents.rs b/src/openhuman/modules/documents.rs index fede77ec0b..62d3f2053e 100644 --- a/src/openhuman/modules/documents.rs +++ b/src/openhuman/modules/documents.rs @@ -24,11 +24,13 @@ //! deadline underneath would make the effective limit the smaller of two numbers //! nobody picked together. +use crate::openhuman::tools::implementations::document::format::spec::{ + DocumentSpec, WirePresentationSpec, +}; use base64::engine::general_purpose::STANDARD as BASE64; use base64::Engine as _; use serde::Deserialize; use tinybus::stream::StreamRef; -use tinydocs::spec::{DocumentSpec, WirePresentationSpec}; use super::{host, ops, registry}; use crate::openhuman::config::Config; diff --git a/src/openhuman/modules/documents_tests.rs b/src/openhuman/modules/documents_tests.rs index 7e97c5eeb6..2fb2649857 100644 --- a/src/openhuman/modules/documents_tests.rs +++ b/src/openhuman/modules/documents_tests.rs @@ -8,7 +8,9 @@ use super::{classify, sha256_hex, DocumentCallError}; use crate::openhuman::config::Config; -use tinydocs::spec::{DocumentSpec, WirePresentationSpec}; +use crate::openhuman::tools::implementations::document::format::spec::{ + DocumentSpec, WirePresentationSpec, +}; /// A config with modules enabled but nothing fetchable. fn offline_config() -> Config { diff --git a/src/openhuman/modules/host.rs b/src/openhuman/modules/host.rs index 7edbb6cd40..9880fcd410 100644 --- a/src/openhuman/modules/host.rs +++ b/src/openhuman/modules/host.rs @@ -27,19 +27,12 @@ //! Everything here is created once and lives for the process. There is no //! shutdown path because there is nothing a shutdown could reclaim. //! -//! # The runtime that gets here first owns the bus -//! -//! The broker and the connection are tokio tasks, so they belong to whichever -//! runtime calls [`runtime`] first. In the core that is the one runtime the -//! process has, and the question never arises. -//! -//! It arises in tests. Two `#[tokio::test]` functions each build their own -//! runtime, and the second one to call a loaded module finds a broker whose tasks -//! died with the first runtime — the call does not fail, it hangs until whatever -//! deadline is above it fires. Any test that drives a real module therefore has -//! to be the only one in its process, which is why the module-backed tool tests -//! are `#[ignore]`d rather than merely gated on an artifact being present. +//! The broker lives on a dedicated process-lifetime Tokio runtime. This keeps a +//! module usable across short-lived caller runtimes (notably `#[tokio::test]`) +//! and also prevents an embedding host from accidentally tying module lifetime +//! to an independently managed application task runtime. +use std::sync::Arc; use std::sync::OnceLock; use tinybus::broker::Broker; @@ -56,6 +49,9 @@ pub struct ModuleRuntime { host: ModuleHost, /// This process's client connection, used to call into loaded modules. connection: Connection, + /// Handle for work that must spawn module transport tasks with process + /// lifetime rather than the caller runtime's lifetime. + handle: tokio::runtime::Handle, } impl ModuleRuntime { @@ -71,6 +67,17 @@ impl ModuleRuntime { &self.connection } + /// Run module admission work on this runtime's blocking pool. + pub async fn blocking(&self, work: F) -> Result<(), String> + where + F: FnOnce() -> Result<(), String> + Send + 'static, + { + self.handle + .spawn_blocking(work) + .await + .map_err(|error| format!("the module loader did not finish: {error}"))? + } + /// A proxy for one object on a loaded module. /// /// # Errors @@ -97,6 +104,47 @@ pub async fn runtime() -> tinybus::Result<&'static ModuleRuntime> { return Ok(existing); } + static START: OnceLock> = OnceLock::new(); + let started = START.get_or_init(|| { + let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1); + std::thread::Builder::new() + .name("openhuman-module-bus".to_string()) + .spawn(move || { + let tokio_runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name("openhuman-module-worker") + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + return; + } + }; + let result = tokio_runtime.block_on(build_runtime()); + match result { + Ok(runtime) => { + let _ = RUNTIME.set(runtime); + let _ = ready_tx.send(Ok(())); + tokio_runtime.block_on(std::future::pending::<()>()); + } + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + } + } + }) + .map_err(|error| error.to_string())?; + ready_rx.recv().map_err(|error| error.to_string())? + }); + started + .as_ref() + .map_err(|error| tinybus::Error::Transport(error.clone()))?; + RUNTIME + .get() + .ok_or_else(|| tinybus::Error::Transport("module runtime did not start".to_string())) +} + +async fn build_runtime() -> tinybus::Result { let transport = MemoryBus::new(); let broker = Broker::new(); // The broker task is deliberately not retained. It lives as long as the @@ -126,10 +174,15 @@ pub async fn runtime() -> tinybus::Result<&'static ModuleRuntime> { let host = ModuleHost::new(broker); let connection = Connection::connect(transport.connect().await?).await?; - let runtime = ModuleRuntime { host, connection }; - // A concurrent caller may have won the race. Its runtime is equivalent, so - // take the winner's and let ours drop. - Ok(RUNTIME.get_or_init(|| runtime)) + if let Some(config) = super::memory::policy().cloned() { + super::memory_host::install(&connection, Arc::clone(&config)).await?; + } + + Ok(ModuleRuntime { + host, + connection, + handle: tokio::runtime::Handle::current(), + }) } /// Whether the module runtime has been stood up. @@ -145,15 +198,7 @@ pub fn is_started() -> bool { mod tests { use super::{is_started, runtime}; - /// Everything that touches the process-global module runtime, in one test. - /// - /// One test and not several, on purpose: `runtime()` is a process-global - /// started by whichever tokio runtime reaches it first, and each - /// `#[tokio::test]` builds its own. A second test function would find a - /// broker whose tasks died with the runtime that spawned it, and its call - /// would hang until something above it timed out rather than failing — the - /// same affinity hazard the module spec documents for the module-backed tool - /// tests. Splitting these up would reintroduce it. + /// The process-global runtime is stable within one caller runtime. #[tokio::test] async fn the_module_bus_is_a_singleton_and_serves_proxies() { let first = runtime().await.expect("runtime should start"); diff --git a/src/openhuman/modules/memory.rs b/src/openhuman/modules/memory.rs index 5bfa168f12..ddcbd8655a 100644 --- a/src/openhuman/modules/memory.rs +++ b/src/openhuman/modules/memory.rs @@ -11,7 +11,7 @@ //! roughly four thousand pre-boot tests invoke with no tokio runtime at all. So //! [`ModuleMemoryProvider::new`] cannot load the module, cannot dial the bus, and //! cannot await anything. It stores its configuration and resolves on first use, -//! the same contract `driver::embedded` follows. +//! the same lazy-loading contract used by the module host. //! //! That has one consequence worth stating plainly, because it looks like a //! shortcut and is not: @@ -21,14 +21,11 @@ //! `MemoryProvider::capabilities` is a **synchronous** method, and the module can //! only answer it over the bus. It therefore cannot be asked here. //! -//! It does not need to be. The module serves exactly the mandatory three — -//! `tinymemory-tinycortex` advertises Core, Recall and Portability because the -//! optional families need a host's configuration, embedding compute and job -//! queue — and that is a property of the artifact's *source*, fixed at the -//! version the registry pins, not something to discover at runtime. So this -//! returns [`Capabilities::mandatory`], and [`ModuleMemoryProvider::verify`] -//! cross-checks it against the module's own answer on first use and logs loudly -//! on disagreement. +//! It does not need to be. The TinyMemory module serves the complete shared API, +//! and that is a property of the artifact's *source*, fixed at the version the +//! registry pins, not something to discover at runtime. So this returns +//! [`Capabilities::all`], and [`ModuleMemoryProvider::verify`] cross-checks it +//! against the module's own answer on first use and logs loudly on disagreement. //! //! Guessing high would be the dangerous direction: the kernel filters its RPC //! surface and agent-tool assembly from this set, so an overstated capability @@ -37,23 +34,36 @@ //! //! # Errors round-trip through the shared table //! -//! `tinymemory_api::wire` maps a `MemoryError` to a `(name, message)` pair and +//! `crate::openhuman::memory::api::wire` maps a `MemoryError` to a `(name, message)` pair and //! back, and **both ends use it**. Reimplementing the mapping here is what would //! let a `PathEscape` arrive as an `Invalid`, silently reclassifying a sandbox //! escape as a caller mistake. use std::sync::Arc; +use crate::openhuman::memory::api::capabilities::Capabilities; +use crate::openhuman::memory::api::chunks::Chunk; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::goals::GoalsDoc; +use crate::openhuman::memory::api::health::MemoryHealth; +use crate::openhuman::memory::api::provider::types::{ + DiffReport, EntityHit, ExportPage, ExportRecord, ImportOutcome, IngestItem, IngestOutcome, + MaintenanceReport, SnapshotRef, SourceItem, SourceScope, +}; +use crate::openhuman::memory::api::provider::{ + MemoryCore, MemoryDiff, MemoryDocuments, MemoryEntities, MemoryGoals, MemoryGraph, + MemoryIngest, MemoryMaintenance, MemoryPortability, MemoryProvider, MemoryRecall, + MemorySourceSink, MemoryToolMemory, MemoryTree, +}; +use crate::openhuman::memory::api::recall::OwnedRecallOpts; +use crate::openhuman::memory::api::tool_memory::ToolMemoryRule; +use crate::openhuman::memory::api::tree::{IngestRequest, QueryResult, TreeStatus}; +use crate::openhuman::memory::api::types::{ + GraphRelationRecord, MemoryCategory, MemoryEntry, MemoryKvRecord, MemoryTaint, + NamespaceDocumentInput, NamespaceRetrievalContext, NamespaceSummary, StoredMemoryDocument, +}; +use crate::openhuman::memory::api::wire; use async_trait::async_trait; -use tinymemory_api::capabilities::Capabilities; -use tinymemory_api::error::MemoryError; -use tinymemory_api::health::MemoryHealth; -use tinymemory_api::provider::mandatory::{MemoryCore, MemoryPortability, MemoryRecall}; -use tinymemory_api::provider::types::{ExportPage, ExportRecord, ImportOutcome, SourceScope}; -use tinymemory_api::provider::MemoryProvider; -use tinymemory_api::recall::OwnedRecallOpts; -use tinymemory_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; -use tinymemory_api::wire; use super::{host, ops, registry}; use crate::openhuman::config::Config; @@ -95,7 +105,7 @@ pub fn set_modules_policy(config: Arc) { } /// The published policy, if boot supplied one. -fn policy() -> Option<&'static Arc> { +pub(crate) fn policy() -> Option<&'static Arc> { MODULES_POLICY.get() } @@ -176,6 +186,13 @@ impl ModuleMemoryProvider { let runtime = host::runtime().await.map_err(|error| { MemoryError::Other(anyhow::anyhow!("the module bus is not running: {error}")) })?; + super::memory_host::install(runtime.connection(), Arc::clone(config)) + .await + .map_err(|error| { + MemoryError::Other(anyhow::anyhow!( + "the memory module host callbacks are unavailable: {error}" + )) + })?; let proxy = runtime .proxy(record.bus_name, record.object_path) .map_err(|error| MemoryError::Other(anyhow::anyhow!(error.to_string())))?; @@ -187,18 +204,16 @@ impl ModuleMemoryProvider { /// Cross-check the module's advertised capabilities against what this build /// assumes, once per process. /// - /// Logged rather than fatal. A module that advertises *more* than the - /// mandatory three is not dangerous — the kernel simply will not use the - /// extra families, because it filtered its surface from the static set — but - /// it does mean the registry pin and the artifact have diverged, which is - /// worth seeing. A module advertising *less* is the real problem, and says so. + /// Logged rather than fatal. Any mismatch means the registry pin and the + /// artifact have diverged; advertising less is the dangerous direction, + /// because the host assembled its full memory surface from this static set. async fn verify(&self, proxy: &tinybus::Proxy) { if self.verified.get().is_some() { return; } match proxy.call::("Capabilities", ()).await { Ok(actual) => { - let assumed = Capabilities::mandatory(); + let assumed = Capabilities::all(); if actual != assumed { log::warn!( "[modules:memory] the module advertises {actual:?} but this build \ @@ -228,9 +243,9 @@ impl MemoryProvider for ModuleMemoryProvider { &self.driver_id } - /// The mandatory three. See the module docs on why this is static. + /// Every family is implemented by the pinned compiled module. fn capabilities(&self) -> Capabilities { - Capabilities::mandatory() + Capabilities::all() } async fn health(&self) -> MemoryHealth { @@ -260,6 +275,37 @@ impl MemoryProvider for ModuleMemoryProvider { .await .map_err(|error| from_bus(&error)) } + + fn as_ingest(&self) -> Option<&dyn MemoryIngest> { + Some(self) + } + fn as_documents(&self) -> Option<&dyn MemoryDocuments> { + Some(self) + } + fn as_tree(&self) -> Option<&dyn MemoryTree> { + Some(self) + } + fn as_entities(&self) -> Option<&dyn MemoryEntities> { + Some(self) + } + fn as_graph(&self) -> Option<&dyn MemoryGraph> { + Some(self) + } + fn as_diff(&self) -> Option<&dyn MemoryDiff> { + Some(self) + } + fn as_goals(&self) -> Option<&dyn MemoryGoals> { + Some(self) + } + fn as_tool_memory(&self) -> Option<&dyn MemoryToolMemory> { + Some(self) + } + fn as_sources(&self) -> Option<&dyn MemorySourceSink> { + Some(self) + } + fn as_maintenance(&self) -> Option<&dyn MemoryMaintenance> { + Some(self) + } } #[async_trait] @@ -383,6 +429,335 @@ impl MemoryPortability for ModuleMemoryProvider { } } +macro_rules! module_call { + ($self:expr, $operation:literal, $method:literal, $args:expr) => { + $self + .proxy($operation) + .await? + .call($method, $args) + .await + .map_err(|error| from_bus(&error)) + }; +} + +#[async_trait] +impl MemoryIngest for ModuleMemoryProvider { + async fn ingest_document(&self, item: IngestItem) -> Result { + module_call!(self, "ingest_document", "IngestDocument", (item,)) + } + async fn ingest_chat(&self, messages: Vec) -> Result { + module_call!(self, "ingest_chat", "IngestChat", (messages,)) + } +} + +#[async_trait] +impl MemoryDocuments for ModuleMemoryProvider { + async fn put_document(&self, input: NamespaceDocumentInput) -> Result { + module_call!(self, "put_document", "PutDocument", (input,)) + } + async fn get_document( + &self, + namespace: &str, + key: &str, + ) -> Result, MemoryError> { + module_call!(self, "get_document", "GetDocument", (namespace, key)) + } + async fn list_documents( + &self, + namespace: Option<&str>, + ) -> Result { + module_call!( + self, + "list_documents", + "ListDocuments", + (namespace.map(str::to_string),) + ) + } + async fn list_namespaces(&self) -> Result, MemoryError> { + module_call!(self, "list_namespaces", "ListNamespaces", ()) + } + async fn delete_document( + &self, + namespace: &str, + document_id: &str, + ) -> Result { + module_call!( + self, + "delete_document", + "DeleteDocument", + (namespace, document_id) + ) + } + async fn clear_namespace(&self, namespace: &str) -> Result<(), MemoryError> { + module_call!(self, "clear_namespace", "ClearNamespace", (namespace,)) + } + async fn query_documents( + &self, + namespace: &str, + query: &str, + limit: usize, + ) -> Result { + module_call!( + self, + "query_documents", + "QueryDocuments", + (namespace, query, limit) + ) + } + async fn recall_documents( + &self, + namespace: &str, + limit: usize, + ) -> Result { + module_call!( + self, + "recall_documents", + "RecallDocuments", + (namespace, limit) + ) + } +} + +#[async_trait] +impl MemoryTree for ModuleMemoryProvider { + async fn append(&self, request: IngestRequest) -> Result<(), MemoryError> { + module_call!(self, "append", "Append", (request,)) + } + async fn query_source( + &self, + namespace: &str, + source_id: &str, + limit: usize, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + module_call!( + self, + "query_source", + "QuerySource", + (namespace, source_id, limit, scope.cloned()) + ) + } + async fn drill_down(&self, namespace: &str, node_id: &str) -> Result { + module_call!(self, "drill_down", "DrillDown", (namespace, node_id)) + } + async fn seal(&self, namespace: &str) -> Result { + module_call!(self, "seal", "Seal", (namespace,)) + } + async fn cascade(&self, namespace: &str) -> Result { + module_call!(self, "cascade", "Cascade", (namespace,)) + } +} + +#[async_trait] +impl MemoryEntities for ModuleMemoryProvider { + async fn entities( + &self, + namespace: &str, + query: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + module_call!( + self, + "entities", + "Entities", + (namespace, query.map(str::to_string), limit) + ) + } + async fn entity_edges( + &self, + namespace: &str, + entity_id: &str, + limit: usize, + ) -> Result, MemoryError> { + module_call!( + self, + "entity_edges", + "EntityEdges", + (namespace, entity_id, limit) + ) + } + async fn touch_entities( + &self, + namespace: &str, + entity_ids: &[String], + ) -> Result<(), MemoryError> { + module_call!( + self, + "touch_entities", + "TouchEntities", + (namespace, entity_ids.to_vec()) + ) + } +} + +#[async_trait] +impl MemoryGraph for ModuleMemoryProvider { + async fn kv_get( + &self, + namespace: Option<&str>, + key: &str, + ) -> Result, MemoryError> { + module_call!( + self, + "kv_get", + "KvGet", + (namespace.map(str::to_string), key) + ) + } + async fn kv_put( + &self, + namespace: Option<&str>, + key: &str, + value: serde_json::Value, + ) -> Result<(), MemoryError> { + module_call!( + self, + "kv_put", + "KvPut", + (namespace.map(str::to_string), key, value) + ) + } + async fn kv_delete(&self, namespace: Option<&str>, key: &str) -> Result { + module_call!( + self, + "kv_delete", + "KvDelete", + (namespace.map(str::to_string), key) + ) + } + async fn kv_list( + &self, + namespace: Option<&str>, + prefix: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + module_call!( + self, + "kv_list", + "KvList", + ( + namespace.map(str::to_string), + prefix.map(str::to_string), + limit + ) + ) + } + async fn relations( + &self, + namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + module_call!( + self, + "relations", + "Relations", + ( + namespace.map(str::to_string), + subject.map(str::to_string), + predicate.map(str::to_string), + limit + ) + ) + } + async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError> { + module_call!(self, "put_relation", "PutRelation", (relation,)) + } +} + +#[async_trait] +impl MemoryDiff for ModuleMemoryProvider { + async fn capture_snapshot(&self, source_id: &str) -> Result { + module_call!(self, "capture_snapshot", "CaptureSnapshot", (source_id,)) + } + async fn snapshots( + &self, + source_id: &str, + limit: usize, + ) -> Result, MemoryError> { + module_call!(self, "snapshots", "Snapshots", (source_id, limit)) + } + async fn diff( + &self, + source_id: &str, + from: Option<&str>, + to: &str, + ) -> Result { + module_call!( + self, + "diff", + "Diff", + (source_id, from.map(str::to_string), to) + ) + } +} + +#[async_trait] +impl MemoryGoals for ModuleMemoryProvider { + async fn goals(&self) -> Result { + module_call!(self, "goals", "Goals", ()) + } + async fn set_goals(&self, goals: GoalsDoc) -> Result<(), MemoryError> { + module_call!(self, "set_goals", "SetGoals", (goals,)) + } +} + +#[async_trait] +impl MemoryToolMemory for ModuleMemoryProvider { + async fn tool_rules(&self, tool_name: &str) -> Result, MemoryError> { + module_call!(self, "tool_rules", "ToolRules", (tool_name,)) + } + async fn put_tool_rule(&self, rule: ToolMemoryRule) -> Result<(), MemoryError> { + module_call!(self, "put_tool_rule", "PutToolRule", (rule,)) + } + async fn delete_tool_rule(&self, tool_name: &str, rule_id: &str) -> Result { + module_call!( + self, + "delete_tool_rule", + "DeleteToolRule", + (tool_name, rule_id) + ) + } +} + +#[async_trait] +impl MemorySourceSink for ModuleMemoryProvider { + async fn accept_source_items( + &self, + source_id: &str, + source_kind: &str, + items: Vec, + taint: MemoryTaint, + ) -> Result { + module_call!( + self, + "accept_source_items", + "AcceptSourceItems", + (source_id, source_kind, items, taint) + ) + } + async fn forget_source(&self, source_id: &str) -> Result { + module_call!(self, "forget_source", "ForgetSource", (source_id,)) + } +} + +#[async_trait] +impl MemoryMaintenance for ModuleMemoryProvider { + async fn reembed(&self) -> Result { + module_call!(self, "reembed", "Reembed", ()) + } + async fn compact(&self) -> Result { + module_call!(self, "compact", "Compact", ()) + } + async fn consolidate(&self) -> Result { + module_call!(self, "consolidate", "Consolidate", ()) + } + async fn doctor(&self) -> Result { + module_call!(self, "doctor", "Doctor", ()) + } +} + #[cfg(test)] #[path = "memory_tests.rs"] mod tests; diff --git a/src/openhuman/modules/memory_host.rs b/src/openhuman/modules/memory_host.rs new file mode 100644 index 0000000000..e2948babda --- /dev/null +++ b/src/openhuman/modules/memory_host.rs @@ -0,0 +1,292 @@ +//! Host-owned callbacks used by the separately compiled TinyMemory module. + +use crate::core::bus::BUS; +use crate::openhuman::config::Config; +use crate::openhuman::memory::api::host::{MemoryEvent, SpacyResponse}; +use std::sync::Arc; +use tinyagents::harness::model::{ModelRequest, ModelResponse}; +use tinybus::ObjectPath; + +const EMBEDDING_NAME: &str = "ai.tinyhumans.tinymemory.EmbeddingHost"; +const EMBEDDING_PATH: &str = "/ai/tinyhumans/tinymemory/EmbeddingHost"; +const CHAT_NAME: &str = "ai.tinyhumans.tinymemory.ChatHost"; +const CHAT_PATH: &str = "/ai/tinyhumans/tinymemory/ChatHost"; +const RUNTIME_NAME: &str = "ai.tinyhumans.tinymemory.RuntimeHost"; +const RUNTIME_PATH: &str = "/ai/tinyhumans/tinymemory/RuntimeHost"; + +#[derive(Clone)] +struct EmbeddingCallbacks(Arc); + +#[tinybus::interface(name = "ai.tinyhumans.tinymemory.EmbeddingHost")] +impl EmbeddingCallbacks { + async fn embed( + &self, + provider: String, + model: String, + dimensions: usize, + texts: Vec, + ) -> tinybus::Result>> { + let api_key = crate::openhuman::inference::embeddings::resolve_api_key(&self.0, &provider); + let endpoint = self + .0 + .cloud_providers + .iter() + .find(|candidate| candidate.slug == provider) + .map(|candidate| candidate.endpoint.as_str()) + .filter(|endpoint| !endpoint.is_empty()); + let embedder = + crate::openhuman::inference::embeddings::create_embedding_provider_with_config( + &self.0, &provider, &model, dimensions, &api_key, endpoint, + ) + .map_err(method_error)?; + let borrowed: Vec<&str> = texts.iter().map(String::as_str).collect(); + embedder.embed(&borrowed).await.map_err(method_error) + } +} + +#[derive(Clone)] +struct ChatCallbacks(Arc); + +#[tinybus::interface(name = "ai.tinyhumans.tinymemory.ChatHost")] +impl ChatCallbacks { + async fn complete( + &self, + role: String, + request: ModelRequest, + ) -> tinybus::Result { + let (model, _) = crate::openhuman::inference::provider::create_chat_model_with_model_id( + &role, + &self.0, + self.0.default_temperature, + ) + .map_err(method_error)?; + model.invoke(&(), request).await.map_err(method_error) + } +} + +#[derive(Clone)] +struct RuntimeCallbacks(Arc); + +#[tinybus::interface(name = "ai.tinyhumans.tinymemory.RuntimeHost")] +impl RuntimeCallbacks { + async fn publish_event(&self, event: MemoryEvent) -> tinybus::Result<()> { + if let Some(event) = into_domain_event(event) { + BUS.publish(event); + } + Ok(()) + } + + async fn report_error( + &self, + classify_expected: bool, + rendered: String, + domain: String, + operation: String, + tags: Vec<(String, String)>, + ) -> tinybus::Result<()> { + let borrowed: Vec<(&str, &str)> = tags + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + .collect(); + if classify_expected { + crate::core::observability::report_error_or_expected( + &rendered, &domain, &operation, &borrowed, + ); + } else { + crate::core::observability::report_error(&rendered, &domain, &operation, &borrowed); + } + Ok(()) + } + + async fn extract_spacy(&self, text: String) -> tinybus::Result { + let response = crate::openhuman::runtime::python_server::extract_spacy(&self.0, &text) + .await + .map_err(method_error)?; + serde_json::from_value(serde_json::to_value(response).map_err(method_error)?) + .map_err(method_error) + } +} + +fn method_error(error: impl std::fmt::Display) -> tinybus::Error { + tinybus::Error::MethodFailed { + name: "ai.tinyhumans.tinymemory.Error.Host".to_string(), + message: error.to_string(), + } +} + +pub(super) async fn install( + connection: &tinybus::Connection, + config: Arc, +) -> tinybus::Result<()> { + static INSTALLED: tokio::sync::OnceCell<()> = tokio::sync::OnceCell::const_new(); + INSTALLED + .get_or_try_init(|| async move { + connection + .serve_at( + ObjectPath::new(EMBEDDING_PATH)?, + EmbeddingCallbacks(Arc::clone(&config)), + ) + .await?; + connection + .serve_at( + ObjectPath::new(CHAT_PATH)?, + ChatCallbacks(Arc::clone(&config)), + ) + .await?; + connection + .serve_at(ObjectPath::new(RUNTIME_PATH)?, RuntimeCallbacks(config)) + .await?; + connection.request_name(EMBEDDING_NAME).await?; + connection.request_name(CHAT_NAME).await?; + connection.request_name(RUNTIME_NAME).await + }) + .await + .map(|_| ()) +} + +fn into_domain_event(event: MemoryEvent) -> Option { + use crate::core::events::DomainEvent; + Some(match event { + MemoryEvent::SyncStageChanged { + trigger, + stage, + provider, + connection_id, + detail, + source_id, + } => DomainEvent::MemorySyncStageChanged { + trigger, + stage, + provider, + connection_id, + detail, + source_id, + }, + MemoryEvent::IngestionStarted { + document_id, + title, + namespace, + queue_depth, + } => DomainEvent::MemoryIngestionStarted { + document_id, + title, + namespace, + queue_depth, + }, + MemoryEvent::IngestionCompleted { + document_id, + namespace, + success, + elapsed_ms, + queue_depth, + } => DomainEvent::MemoryIngestionCompleted { + document_id, + namespace, + success, + elapsed_ms, + queue_depth, + }, + MemoryEvent::DocumentCanonicalized { + source_id, + source_kind, + chunks_written, + chunk_ids, + canonicalized_at, + body_preview, + } => DomainEvent::DocumentCanonicalized { + source_id, + source_kind, + chunks_written, + chunk_ids, + canonicalized_at, + body_preview, + }, + MemoryEvent::TreeSummarizerHourCompleted { + namespace, + node_id, + token_count, + } => DomainEvent::TreeSummarizerHourCompleted { + namespace, + node_id, + token_count, + }, + MemoryEvent::TreeSummarizerPropagated { + namespace, + node_id, + level, + token_count, + } => DomainEvent::TreeSummarizerPropagated { + namespace, + node_id, + level, + token_count, + }, + MemoryEvent::TreeSummarizerRebuildCompleted { + namespace, + total_nodes, + } => DomainEvent::TreeSummarizerRebuildCompleted { + namespace, + total_nodes, + }, + MemoryEvent::TreeBuildProgress { + phase, + step, + tree_scope, + level, + item_count, + detail, + } => DomainEvent::MemoryTreeBuildProgress { + phase, + step, + tree_scope, + level, + item_count, + detail, + }, + MemoryEvent::EmbeddingModelUnhealthy(reason) => DomainEvent::EmbeddingModelUnhealthy { + provider: reason.provider, + model: reason.model, + fallback_provider: reason.fallback_provider, + message: reason.message, + }, + MemoryEvent::DriverBindFailed { + configured_driver, + bound_driver, + reason, + } => DomainEvent::MemoryDriverBindFailed { + configured_driver, + bound_driver, + reason, + }, + MemoryEvent::DiffSnapshotTaken { + snapshot_id, + source_id, + source_kind, + item_count, + trigger, + } => DomainEvent::MemoryDiffSnapshotTaken { + snapshot_id, + source_id, + source_kind, + item_count, + trigger, + }, + MemoryEvent::DiffMarkedRead { + source_ids, + snapshot_ids, + } => DomainEvent::MemoryDiffMarkedRead { + source_ids, + snapshot_ids, + }, + MemoryEvent::ComposioIntegrationsChanged { toolkits } => { + DomainEvent::ComposioIntegrationsChanged { toolkits } + } + MemoryEvent::SyncRequested { channel_id } => { + DomainEvent::MemorySyncRequested { channel_id } + } + MemoryEvent::LocalModelUnavailable { origin } => { + crate::openhuman::memory::tree::health::user_error::publish_local_model_unavailable_user_error(&origin); + return None; + } + }) +} diff --git a/src/openhuman/modules/memory_tests.rs b/src/openhuman/modules/memory_tests.rs index 1a2b419c70..5d8aed51ec 100644 --- a/src/openhuman/modules/memory_tests.rs +++ b/src/openhuman/modules/memory_tests.rs @@ -9,9 +9,9 @@ use std::sync::Arc; -use tinymemory_api::capabilities::{Capabilities, Capability}; -use tinymemory_api::error::MemoryError; -use tinymemory_api::provider::MemoryProvider; +use crate::openhuman::memory::api::capabilities::{Capabilities, Capability}; +use crate::openhuman::memory::api::error::MemoryError; +use crate::openhuman::memory::api::provider::MemoryProvider; use super::{from_bus, ModuleMemoryProvider, MODULE_ID}; use crate::openhuman::config::Config; @@ -43,20 +43,16 @@ fn construction_touches_no_io_and_needs_no_runtime() { } #[test] -fn the_advertised_capabilities_are_exactly_the_mandatory_three() { - // Must match what the module serves. Overstating is the dangerous direction: - // the kernel filters its RPC surface and agent-tool list from this set, so an - // extra family registers methods that answer errors. +fn the_advertised_capabilities_cover_the_complete_memory_api() { + // The compiled module owns the complete TinyMemory API, so the host can + // assemble every memory RPC and tool family before the async bus starts. let capabilities = provider().capabilities(); - assert_eq!(capabilities, Capabilities::mandatory()); + assert_eq!(capabilities, Capabilities::all()); for mandatory in Capability::MANDATORY { assert!(capabilities.contains(mandatory), "{mandatory:?} is missing"); } - assert!( - !capabilities.contains(Capability::Tree), - "an optional family the module cannot serve must not be advertised" - ); + assert!(capabilities.contains(Capability::Tree)); } #[test] @@ -103,13 +99,13 @@ fn the_memory_record_publishes_one_asset_per_supported_host() { fn a_not_found_survives_the_round_trip_as_not_found() { // `get`'s contract makes a missing entry `Ok(None)` and an `Invalid` a real // failure, so collapsing the two would be observable to a caller. - let error = from_bus(&failure(tinymemory_api::wire::NOT_FOUND)); + let error = from_bus(&failure(crate::openhuman::memory::api::wire::NOT_FOUND)); assert!(matches!(error, MemoryError::NotFound(_)), "{error:?}"); } #[test] fn an_invalid_input_is_reported_as_something_the_caller_can_fix() { - let error = from_bus(&failure(tinymemory_api::wire::INVALID)); + let error = from_bus(&failure(crate::openhuman::memory::api::wire::INVALID)); assert!(matches!(error, MemoryError::Invalid(_)), "{error:?}"); } @@ -117,13 +113,13 @@ fn an_invalid_input_is_reported_as_something_the_caller_can_fix() { fn a_path_escape_does_not_arrive_as_a_caller_mistake() { // The mapping's most security-relevant case: a sandbox escape must not be // reclassified as a malformed argument. - let error = from_bus(&failure(tinymemory_api::wire::PATH_ESCAPE)); + let error = from_bus(&failure(crate::openhuman::memory::api::wire::PATH_ESCAPE)); assert!(matches!(error, MemoryError::PathEscape(_)), "{error:?}"); } #[test] fn an_unsupported_capability_keeps_its_family_name() { - let error = from_bus(&failure(tinymemory_api::wire::UNSUPPORTED)); + let error = from_bus(&failure(crate::openhuman::memory::api::wire::UNSUPPORTED)); assert!( matches!(error, MemoryError::Unsupported { .. }), "{error:?}" @@ -164,7 +160,10 @@ async fn a_disabled_host_reports_down_rather_than_erroring() { let provider = ModuleMemoryProvider::new(Arc::new(config)); let health = provider.health().await; assert!( - matches!(health, tinymemory_api::health::MemoryHealth::Down { .. }), + matches!( + health, + crate::openhuman::memory::api::health::MemoryHealth::Down { .. } + ), "a disabled module host must report Down, got {health:?}" ); } @@ -176,7 +175,8 @@ async fn a_call_against_a_disabled_host_fails_instead_of_hanging() { let provider = ModuleMemoryProvider::new(Arc::new(config)); let outcome = - tinymemory_api::provider::mandatory::MemoryCore::get(&provider, "ns", "key").await; + crate::openhuman::memory::api::provider::mandatory::MemoryCore::get(&provider, "ns", "key") + .await; assert!(outcome.is_err(), "expected an error, got {outcome:?}"); } diff --git a/src/openhuman/modules/mod.rs b/src/openhuman/modules/mod.rs index e848fa13f9..abb7e5b013 100644 --- a/src/openhuman/modules/mod.rs +++ b/src/openhuman/modules/mod.rs @@ -41,6 +41,7 @@ pub mod boot; pub mod documents; pub mod host; pub mod memory; +mod memory_host; pub mod ops; pub mod platform; pub mod registry; diff --git a/src/openhuman/modules/ops.rs b/src/openhuman/modules/ops.rs index be37d63d58..c5d5d6f596 100644 --- a/src/openhuman/modules/ops.rs +++ b/src/openhuman/modules/ops.rs @@ -119,12 +119,14 @@ async fn resolve(config: &Config, record: &'static ModuleRecord) -> Result<(), S // An override points at a developer's own build. Checked before the pinned // release so a module can be iterated on against a live core. if let Some(path) = local_override(config, record.id) { - return blocking(move || load_local(runtime, &path, record.id)).await; + let module_config = module_config(config, record.id); + return blocking(move || load_local(runtime, &path, record.id, module_config)).await; } // An artifact already extracted into the install directory by an earlier run. if let Some(path) = installed_artifact(config, record) { - return blocking(move || load_local(runtime, &path, record.id)).await; + let module_config = module_config(config, record.id); + return blocking(move || load_local(runtime, &path, record.id, module_config)).await; } // The search path tinybus itself honours, including OPENHUMAN_MODULE_PATH. @@ -158,7 +160,8 @@ async fn resolve(config: &Config, record: &'static ModuleRecord) -> Result<(), S // hashes the archive, extracts it and `dlopen`s the result — all // synchronously. Left inline it would stall every other task sharing this // worker for the length of a download on whatever link the user has. - blocking(move || download(runtime, record)).await + let module_config = module_config(config, record.id); + blocking(move || download(runtime, record, module_config)).await } /// Run a blocking module operation on the blocking pool. @@ -169,19 +172,23 @@ async fn blocking(work: F) -> Result<(), String> where F: FnOnce() -> Result<(), String> + Send + 'static, { - match tokio::task::spawn_blocking(work).await { - Ok(result) => result, - Err(err) => Err(format!( - "the module loader did not finish: {err}. This is terminal for the running \ - process; restart the app to try again" - )), - } + host::runtime() + .await + .map_err(|error| format!("the module bus could not start: {error}"))? + .blocking(work) + .await + .map_err(|error| { + format!( + "{error}. This is terminal for the running process; restart the app to try again" + ) + }) } /// Download, verify, and load the pinned release artifact for this host. fn download( runtime: &'static host::ModuleRuntime, record: &'static ModuleRecord, + module_config: serde_json::Value, ) -> Result<(), String> { let candidates = platform::host_candidates(); let assets: Vec<_> = candidates @@ -205,7 +212,7 @@ fn download( record.release_url, asset.archive, Some(asset.sha256), - serde_json::json!({}), + module_config.clone(), ) { Ok(_) => { log::info!( @@ -240,8 +247,9 @@ pub(super) fn load_local( runtime: &host::ModuleRuntime, path: &Path, id: &str, + module_config: serde_json::Value, ) -> Result<(), String> { - match runtime.host().load_file(path) { + match runtime.host().load_file_with_config(path, module_config) { Ok(_) => { log::info!("[modules] loaded '{id}' from a local artifact"); Ok(()) @@ -253,6 +261,36 @@ pub(super) fn load_local( } } +/// Configuration crossing into a first-party compiled module. +/// +/// Credentials are intentionally absent. TinyMemory calls back into the host +/// for embedding and chat compute; the other modules need no host config. +fn module_config(config: &Config, id: &str) -> serde_json::Value { + if id != super::memory::MODULE_ID { + return serde_json::json!({}); + } + serde_json::json!({ + "workspace_dir": config.workspace_dir, + "memory": config.memory, + "memory_tree": config.memory_tree, + "scheduler_gate": config.scheduler_gate, + "local_ai": config.local_ai, + "embeddings_provider": config.embeddings_provider, + "memory_provider": config.memory_provider, + "default_model": config.default_model, + "default_temperature": config.default_temperature, + "output_language": config.output_language, + "memory_sources": config.memory_sources, + "embedding_routes": config.embedding_routes, + "storage_provider": config.storage.provider.config, + "ollama_base_url": crate::openhuman::inference::local::ollama_base_url_from_config(config), + "cloud_embedding_model": config.memory.embedding_model, + "cloud_embedding_dimensions": config.memory.embedding_dimensions, + "models_supporting_dimensions": [], + "driver_id": "tinymemory", + }) +} + /// A configured local artifact for `id`, if one is set. fn local_override(config: &Config, id: &str) -> Option { config diff --git a/src/openhuman/modules/registry.rs b/src/openhuman/modules/registry.rs index 4bdb9131d4..cfea6b04b1 100644 --- a/src/openhuman/modules/registry.rs +++ b/src/openhuman/modules/registry.rs @@ -35,63 +35,63 @@ const TINYDOCS: ModuleRecord = ModuleRecord { description: "Document synthesis (.docx, .pptx) and PDF text extraction", bus_name: "ai.tinyhumans.tinydocs.Documents", object_path: "/ai/tinyhumans/tinydocs/Documents", - version: "0.1.12", - release_url: "https://github.com/tinyhumansai/tinydocs/releases/tag/v0.1.12", + version: "0.1.13", + release_url: "https://github.com/tinyhumansai/tinydocs/releases/tag/v0.1.13", assets: &[ PlatformAsset { host_key: "ubuntu-24.04-x86_64", - archive: "tinydocs-module-0.1.12-ubuntu-24.04-x86_64.tar.gz", - sha256: "89a1c6f3ff386a2190bfa4efbef75d564651f75cd8136c8940ec4de950f69a05", + archive: "tinydocs-module-0.1.13-ubuntu-24.04-x86_64.tar.gz", + sha256: "43ad43b0fea00de3f82f960c5eae297b528334780905286f683857cbd7e7fa07", }, PlatformAsset { host_key: "ubuntu-24.04-arm64", - archive: "tinydocs-module-0.1.12-ubuntu-24.04-arm64.tar.gz", - sha256: "685b38dbb9b5beba0105b2991212882ba2d4cb74fa1f3613c9eb9b75de023f0b", + archive: "tinydocs-module-0.1.13-ubuntu-24.04-arm64.tar.gz", + sha256: "66a4d9a4cb1caea86fe6203cde54db06165d483c59e8f86b61439f257be7dff8", }, PlatformAsset { host_key: "ubuntu-22.04-x86_64", - archive: "tinydocs-module-0.1.12-ubuntu-22.04-x86_64.tar.gz", - sha256: "35ac3d05202dfcb425c3d6448f1740656b5df3e6276ecd97ded973f92c356591", + archive: "tinydocs-module-0.1.13-ubuntu-22.04-x86_64.tar.gz", + sha256: "3e3a7c2e774d75654a7e9074e41ad972a670f2a0dcf8ee2648dfdbb404edc7cb", }, PlatformAsset { host_key: "ubuntu-22.04-arm64", - archive: "tinydocs-module-0.1.12-ubuntu-22.04-arm64.tar.gz", - sha256: "3870486bd42fc729cc56b7dae9343aaa854de2b24d30f4a6386a8083db6ef32e", + archive: "tinydocs-module-0.1.13-ubuntu-22.04-arm64.tar.gz", + sha256: "12f0c83a6239423be9001ec57cf9d53a50c639e3d67449646f48a9eef207f36b", }, PlatformAsset { host_key: "macos-26-arm64", - archive: "tinydocs-module-0.1.12-macos-26-arm64.tar.gz", - sha256: "18ab086bd58d8fec2ac407981f2013d7284a8d7e0c07cdc51ee6fdde4535f431", + archive: "tinydocs-module-0.1.13-macos-26-arm64.tar.gz", + sha256: "6a8edb36258a241c62497dd962c3690f0f287944663a7edc00602e652ac72298", }, PlatformAsset { host_key: "macos-26-x86_64", - archive: "tinydocs-module-0.1.12-macos-26-x86_64.tar.gz", - sha256: "426711799118bae95a691d6a61920c4bc93b76e930cdbce4209e730aa8b9efa2", + archive: "tinydocs-module-0.1.13-macos-26-x86_64.tar.gz", + sha256: "dfcd0f79f6ea9ffd7c9f510f4007285a0cc7d434ddf286a9dc870468003d3784", }, PlatformAsset { host_key: "macos-15-arm64", - archive: "tinydocs-module-0.1.12-macos-15-arm64.tar.gz", - sha256: "f0aa5d7076a1ce3cdf4c0cf4dd15e274bfbd7d4ccfced6793e55651a7499f3d4", + archive: "tinydocs-module-0.1.13-macos-15-arm64.tar.gz", + sha256: "8b1be8ac2db781fd0ff8af8815e6dd408d79fd8c489032358447434a21bdf52a", }, PlatformAsset { host_key: "macos-15-x86_64", - archive: "tinydocs-module-0.1.12-macos-15-x86_64.tar.gz", - sha256: "9fbc1aa2dfabe35e492aa6abea90515ab83ea13a191c87306401a499d432e5e3", + archive: "tinydocs-module-0.1.13-macos-15-x86_64.tar.gz", + sha256: "c84dcf6b3fc4eac5985b56297e35eb730dc86c7717fdfe72886f9c189efc22ba", }, PlatformAsset { host_key: "windows-2025-x86_64", - archive: "tinydocs-module-0.1.12-windows-2025-x86_64.zip", - sha256: "4870bb1084ad0435b44d1ec845c5d2f398e27e430bec00ec0f58e0664e5bfc3f", + archive: "tinydocs-module-0.1.13-windows-2025-x86_64.zip", + sha256: "30a0ef74959029ed385ee4a3e47f8f42bd4eeeb12c2d95030107fa7ac16d5dbe", }, PlatformAsset { host_key: "windows-2022-x86_64", - archive: "tinydocs-module-0.1.12-windows-2022-x86_64.zip", - sha256: "f1fc72690dd59890d7a629002ab2ade0547b2a3ca23c5cc54f5d40ed0e8b24af", + archive: "tinydocs-module-0.1.13-windows-2022-x86_64.zip", + sha256: "f8a7097166074aff712e6847207c112f3afcc95a6a875177bcc167b46cd6d332", }, PlatformAsset { host_key: "windows-11-arm64", - archive: "tinydocs-module-0.1.12-windows-11-arm64.zip", - sha256: "c4f7bda63c17a5bbdb10d8e5bab04c9b0113fbd420f83465208b64a286f2127a", + archive: "tinydocs-module-0.1.13-windows-11-arm64.zip", + sha256: "366f92165c1a3ef4361568edacb0ca4053a0209efbf804730ab35ee37b743ee7", }, ], load: LoadPolicy::Lazy, @@ -111,145 +111,132 @@ const TINYWALLET: ModuleRecord = ModuleRecord { description: "Transaction building and assembly for Bitcoin, EVM, Solana and Tron", bus_name: "ai.tinyhumans.tinywallet.Wallet", object_path: "/ai/tinyhumans/tinywallet/Wallet", - version: "0.2.0", - release_url: "https://github.com/tinyhumansai/tinywallet/releases/tag/v0.2.0", + version: "0.2.1", + release_url: "https://github.com/tinyhumansai/tinywallet/releases/tag/v0.2.1", assets: &[ PlatformAsset { host_key: "ubuntu-24.04-x86_64", - archive: "tinywallet-module-0.2.0-ubuntu-24.04-x86_64.tar.gz", - sha256: "827ae2721f4173f76247d7728c1383bd54a590608abb994a0ca2b4742ff2bd85", + archive: "tinywallet-module-0.2.1-ubuntu-24.04-x86_64.tar.gz", + sha256: "42e3440d367c251d687505115b7c0e2bdcd8ec4f064438a03762bbb5c1b651c0", }, PlatformAsset { host_key: "ubuntu-24.04-arm64", - archive: "tinywallet-module-0.2.0-ubuntu-24.04-arm64.tar.gz", - sha256: "5e014a6eca418c94d85f333bd804853a115ab26552e29ee6c779bb87497b116b", + archive: "tinywallet-module-0.2.1-ubuntu-24.04-arm64.tar.gz", + sha256: "9b9f102e1bde35bc59ca9898c0531de87dc34408a6d495a56d821e173284f65a", }, PlatformAsset { host_key: "ubuntu-22.04-x86_64", - archive: "tinywallet-module-0.2.0-ubuntu-22.04-x86_64.tar.gz", - sha256: "be87ddf38ee1c2033fd568d65b22e7171c4dc24ee264cade62e20632bc5defc1", + archive: "tinywallet-module-0.2.1-ubuntu-22.04-x86_64.tar.gz", + sha256: "77c3f8a188ac69d4faa4b7305c83065682b36683754e3072e8f6a1d64c8fe795", }, PlatformAsset { host_key: "ubuntu-22.04-arm64", - archive: "tinywallet-module-0.2.0-ubuntu-22.04-arm64.tar.gz", - sha256: "393e68fc9a5184b3b0d71731ed066e1a0e06186aa59fe5fb8ab4a5f60cca833c", + archive: "tinywallet-module-0.2.1-ubuntu-22.04-arm64.tar.gz", + sha256: "1a63b576eb5f07dd54cfd27f63f3f6c86718fb93e11726ec6482d7cc95db6863", }, PlatformAsset { host_key: "macos-26-arm64", - archive: "tinywallet-module-0.2.0-macos-26-arm64.tar.gz", - sha256: "ce87e1c3b4e6bbb2d41735a8bb10001001d0555cb7841fec09df5fb4d0bd99a4", + archive: "tinywallet-module-0.2.1-macos-26-arm64.tar.gz", + sha256: "3c89b41511156ced51267da77a83519d7fae0ab10b2efa311d1a5ffde748a360", }, PlatformAsset { host_key: "macos-26-x86_64", - archive: "tinywallet-module-0.2.0-macos-26-x86_64.tar.gz", - sha256: "bc700a2993c403140e262e82e74dafc58d71a0b252a0e7c8aa5aa3c5f81cf55d", + archive: "tinywallet-module-0.2.1-macos-26-x86_64.tar.gz", + sha256: "eb3ea578e0b05f03a8150af07de40b4c61b584e0d1c1944b2172bde8a356701c", }, PlatformAsset { host_key: "macos-15-arm64", - archive: "tinywallet-module-0.2.0-macos-15-arm64.tar.gz", - sha256: "6e61acdd6afa48efc72069e25bf9c918905e960522e60244db9e738db8f0207a", + archive: "tinywallet-module-0.2.1-macos-15-arm64.tar.gz", + sha256: "768a6eb74ceff9ddcc6c7d0c79dc2942e29d4264a7df11d82152c921b426aa5a", }, PlatformAsset { host_key: "macos-15-x86_64", - archive: "tinywallet-module-0.2.0-macos-15-x86_64.tar.gz", - sha256: "a63da64043fc960ed13747c30ff6e8e396ba0cba7099b0616b9c6dbc54cb4a8d", + archive: "tinywallet-module-0.2.1-macos-15-x86_64.tar.gz", + sha256: "c9ee8e0367beb56ef9aafbe4d830c4e4a58b4b6d0150f7f42c4fb4536441099c", }, PlatformAsset { host_key: "windows-2025-x86_64", - archive: "tinywallet-module-0.2.0-windows-2025-x86_64.zip", - sha256: "bd4b7156dd031ce1821d563759fad52e73aad30aa9e8388ec0d486a8b804161e", + archive: "tinywallet-module-0.2.1-windows-2025-x86_64.zip", + sha256: "ef64bd36086fcba105f30703bc3ef7102a24a48e510e622c502b8b9bfa2ea68f", }, PlatformAsset { host_key: "windows-2022-x86_64", - archive: "tinywallet-module-0.2.0-windows-2022-x86_64.zip", - sha256: "8fbc5438adb86078b1ea4d0c6a4223daf298029e2f5db2dcd5666d446d9d6dd8", + archive: "tinywallet-module-0.2.1-windows-2022-x86_64.zip", + sha256: "1d9f18071ee185a8b13ffc6c93e0f83c232eac879dca46bb5d36d9e832a133e4", }, PlatformAsset { host_key: "windows-11-arm64", - archive: "tinywallet-module-0.2.0-windows-11-arm64.zip", - sha256: "59e705e458248e8225a9dd5a103b6165e3559699921c7ff78e5b26b938e779a2", + archive: "tinywallet-module-0.2.1-windows-11-arm64.zip", + sha256: "20ed7d288fcd9d8a58eb774a30099a292bff06bc1d5b983a06b88d555e0b41dc", }, ], load: LoadPolicy::Lazy, }; -/// The memory engine, served as a driver. -/// -/// # The digests below came from the release, not from a local build -/// -/// Every `sha256` here was copied verbatim from -/// `v0.3.0`'s `checksum.toml`. Recomputing one from a local -/// build would pin whatever this machine happened to produce, which defeats the -/// point: tinybus fetches the release's own manifest, compares it with this -/// value, hashes the download, and only then extracts. Pinning here is the half -/// of that check which is auditable offline, and the half that makes a release -/// re-cut under the same tag stop matching rather than silently replacing what -/// runs in-process. -/// -/// If these ever need to change, take the new values from the release. Do not -/// run `sha256sum` on `target/release/`. +/// The complete TinyMemory engine, loaded eagerly so its capabilities are +/// available when the kernel assembles its RPC and tool surfaces. const TINYMEMORY: ModuleRecord = ModuleRecord { id: "tinymemory", description: "Local memory engine: store, ranked recall, and portable export", bus_name: "ai.tinyhumans.tinymemory.Memory", object_path: "/ai/tinyhumans/tinymemory/Memory", - version: "0.3.0", - release_url: "https://github.com/tinyhumansai/tinymemory/releases/tag/v0.3.0", + version: "1.0.1", + release_url: "https://github.com/tinyhumansai/tinymemory/releases/tag/v1.0.1", assets: &[ PlatformAsset { host_key: "ubuntu-24.04-x86_64", - archive: "tinymemory-module-0.3.0-ubuntu-24.04-x86_64.tar.gz", - sha256: "c0406030c7cc09b386bc6040ab29dbafe6dd4f428db9623819c567b4c71afb5a", + archive: "tinymemory-module-1.0.1-ubuntu-24.04-x86_64.tar.gz", + sha256: "723bb5b006c6f45258e3176a944beb70524f884eb1abe21e4c7a8747058e32a3", }, PlatformAsset { host_key: "ubuntu-24.04-arm64", - archive: "tinymemory-module-0.3.0-ubuntu-24.04-arm64.tar.gz", - sha256: "c4e165b68887874acba691dd6a103feea8d0acb401a1d8d00c073bc27404d08f", + archive: "tinymemory-module-1.0.1-ubuntu-24.04-arm64.tar.gz", + sha256: "34af9da10d143c5f5ddc13f969da2119bfe89b309acab5bdc2a9c24c7f341706", }, PlatformAsset { host_key: "ubuntu-22.04-x86_64", - archive: "tinymemory-module-0.3.0-ubuntu-22.04-x86_64.tar.gz", - sha256: "e3ac3eb98997bd25b02bdb71c3d565b2e42a89fa4b7f6369758b3ec6a684c857", + archive: "tinymemory-module-1.0.1-ubuntu-22.04-x86_64.tar.gz", + sha256: "35d13463041f455bebd833a71ba370d891a4e59d1b8e576e81635e2777a0c3dd", }, PlatformAsset { host_key: "ubuntu-22.04-arm64", - archive: "tinymemory-module-0.3.0-ubuntu-22.04-arm64.tar.gz", - sha256: "bf06c9540bed6f2c1b95c57f324fd6a33a33bc10a2a7deaddd36fbafc7d02c40", + archive: "tinymemory-module-1.0.1-ubuntu-22.04-arm64.tar.gz", + sha256: "39f4cad7d781e3feb30dead75bccc3b0c432f095a88f4a73a3fa2d40db08c861", }, PlatformAsset { host_key: "macos-26-arm64", - archive: "tinymemory-module-0.3.0-macos-26-arm64.tar.gz", - sha256: "2d4266d1430c7fa9ec3e609785d534a0fe84a4a1d00ae9fbb151c82c32eccf19", + archive: "tinymemory-module-1.0.1-macos-26-arm64.tar.gz", + sha256: "de27f5eb1510e10c558f4856448eb0c07b6f79d0b345be777a299aa334259245", }, PlatformAsset { host_key: "macos-26-x86_64", - archive: "tinymemory-module-0.3.0-macos-26-x86_64.tar.gz", - sha256: "11b63f0492a9c365ebecfc2e7e433cd133b58e0e727322868b6ef3e6edbd8b4b", + archive: "tinymemory-module-1.0.1-macos-26-x86_64.tar.gz", + sha256: "a9bceaf3a9f72708bd0ee8a3ed10c0bd25c01d13296265cb12382cf9ad443f10", }, PlatformAsset { host_key: "macos-15-arm64", - archive: "tinymemory-module-0.3.0-macos-15-arm64.tar.gz", - sha256: "7a39d272aa29148ef652727bbc05d67c361c4d706a7d89a5dedab192bbb75642", + archive: "tinymemory-module-1.0.1-macos-15-arm64.tar.gz", + sha256: "dcb0d7ce49b769f51f231fb864975898c4ba94a53f1d38d53925cabb843e1efd", }, PlatformAsset { host_key: "macos-15-x86_64", - archive: "tinymemory-module-0.3.0-macos-15-x86_64.tar.gz", - sha256: "0c7eba4e64009eec8e1ae19bfc9b5859136ebf6bc6cb7efa7510a61da42f4456", + archive: "tinymemory-module-1.0.1-macos-15-x86_64.tar.gz", + sha256: "cdd4fc69898cc526461aaabbd5b75ddca2d2b180873c079682ca1c4e77442b89", }, PlatformAsset { host_key: "windows-2025-x86_64", - archive: "tinymemory-module-0.3.0-windows-2025-x86_64.zip", - sha256: "4e55458bc79e1c504d1224afb70edd8404fe8ee3bf0f5e2d44bb258cae79fe05", + archive: "tinymemory-module-1.0.1-windows-2025-x86_64.zip", + sha256: "972d15fa8cc3fe401d25a26b8bcc349470831cec5a6e7ab60a546a7f3f130887", }, PlatformAsset { host_key: "windows-2022-x86_64", - archive: "tinymemory-module-0.3.0-windows-2022-x86_64.zip", - sha256: "608354bf847ee32f90fa6f76c918c20441be9ec09544338867512dbf49b4685d", + archive: "tinymemory-module-1.0.1-windows-2022-x86_64.zip", + sha256: "c9c08ba1ee9c60484fc775b54a16954d544b71bbb96e1b604ccdd40327aceec9", }, PlatformAsset { host_key: "windows-11-arm64", - archive: "tinymemory-module-0.3.0-windows-11-arm64.zip", - sha256: "b7b55d5a473d43053e791eb94116e9e88fc3533b8a7325c45dad832018f02766", + archive: "tinymemory-module-1.0.1-windows-11-arm64.zip", + sha256: "47e956ce93ed4f8cea5ab74cc54eb277ac84ebc0ff9939e2b1003f1a0d009a73", }, ], // Eager, unlike the two codecs above. A codec that is never asked for should @@ -417,25 +404,6 @@ mod tests { ("windows", "aarch64", None), ]; for record in ALL { - // A record with no assets at all is a module whose release has not - // been cut yet. It is exempt from the per-host check — there is - // nothing to be missing — but only if it is on this list, so an - // *accidentally* asset-less record still fails here rather than - // silently becoming unsupported on every platform. - // - // Digests must be copied verbatim from the release `checksum.toml`, - // so shipping placeholder assets to satisfy this test would be worse - // than the exemption. - if record.assets.is_empty() { - assert!( - PENDING_RELEASE.contains(&record.id), - "{} publishes no assets and is not a known pre-release module; \ - either add its assets (digests copied from the release \ - checksum.toml) or add it to PENDING_RELEASE with a reason", - record.id - ); - continue; - } for (os, arch, glibc) in hosts { for key in candidates_for(os, arch, glibc) { assert!( @@ -448,19 +416,6 @@ mod tests { } } - /// Modules registered before their first release exists. - /// - /// Being here means `modules.status` reports `Unsupported` on every platform - /// and `ensure_loaded` refuses, which is the correct behaviour for an artifact - /// that cannot be verified because it has not been published. Removing an - /// entry is the last step of a module port. - /// - /// Empty today: every registered module has a release. Kept rather than - /// deleted because the exemption it grants is what lets the check above stay - /// strict — an accidentally asset-less record must fail, and it can only be - /// told apart from a deliberate one by a list like this. - const PENDING_RELEASE: &[&str] = &[]; - #[test] fn find_resolves_known_ids_only() { assert!(find("tinydocs").is_some()); diff --git a/src/openhuman/modules/wallet.rs b/src/openhuman/modules/wallet.rs index 664586c6be..5b3cb004eb 100644 --- a/src/openhuman/modules/wallet.rs +++ b/src/openhuman/modules/wallet.rs @@ -41,7 +41,7 @@ //! with which it is, and [`sign_payload`] dispatches on the tag rather than on //! the chain — so a chain that changes scheme cannot silently sign wrongly. -use tinywallet::wire::{ +use crate::openhuman::web3::wallet::primitives::wire::{ AttachRequest, PublicKey, Scheme, Signature, SignedTransaction, SigningPayload, SigningRequest, TransactionSpec, UnsignedTransaction, }; @@ -145,6 +145,7 @@ pub async fn sign_transaction( /// Dispatches on the payload's own tag, never on the chain: the module is the /// authority on what it needs signed and how, and a host that decided for itself /// would sign wrongly the moment the two disagreed. +#[allow(unreachable_patterns)] fn sign_payload(payload: &SigningPayload, secret: &[u8]) -> Result { let bytes = unhex(&payload.bytes_hex)?; match payload.scheme { diff --git a/src/openhuman/modules/wallet_tests.rs b/src/openhuman/modules/wallet_tests.rs index ba784a1e77..89d651c728 100644 --- a/src/openhuman/modules/wallet_tests.rs +++ b/src/openhuman/modules/wallet_tests.rs @@ -7,8 +7,10 @@ //! themselves are covered where they can be honest: `tinywallet`'s own loader //! E2E, which drives a real module over a real broker. -use tinywallet::wire::{Scheme, Signature, SigningPayload, TransactionSpec}; -use tinywallet::Chain; +use crate::openhuman::web3::wallet::primitives::wire::{ + Scheme, Signature, SigningPayload, TransactionSpec, +}; +use crate::openhuman::web3::wallet::primitives::Chain; use super::{classify, sign_payload, WalletCallError}; use crate::openhuman::config::Config; @@ -34,7 +36,7 @@ fn failure(name: &str) -> tinybus::Error { } fn evm_secret() -> Vec { - tinywallet::key::derive(Chain::Evm, VECTOR, "m/44'/60'/0'/0/0") + crate::openhuman::web3::wallet::primitives::key::derive(Chain::Evm, VECTOR, "m/44'/60'/0'/0/0") .expect("the vector mnemonic derives") .secret_bytes() .to_vec() @@ -151,7 +153,12 @@ fn an_ed25519_payload_is_signed_over_the_whole_message() { // against the public key rather than merely checked for a length. use ed25519_dalek::{Signature as EdSignature, SigningKey, Verifier as _}; - let derived = tinywallet::key::derive(Chain::Solana, VECTOR, "m/44'/501'/0'/0'").unwrap(); + let derived = crate::openhuman::web3::wallet::primitives::key::derive( + Chain::Solana, + VECTOR, + "m/44'/501'/0'/0'", + ) + .unwrap(); let secret = derived.secret_bytes(); let message = b"a solana message that is clearly longer than thirty-two bytes"; diff --git a/src/openhuman/tools/impl/document/engine.rs b/src/openhuman/tools/impl/document/engine.rs index 44d75d3879..be3c509edc 100644 --- a/src/openhuman/tools/impl/document/engine.rs +++ b/src/openhuman/tools/impl/document/engine.rs @@ -1,18 +1,14 @@ -//! Async wrapper around the vendored [`tinydocs`] `.docx` writer. +//! Async host policy around the document module's `.docx` writer. //! -//! The OOXML synthesis itself lives in -//! [`tinydocs::docx::generate`](https://github.com/tinyhumansai/tinydocs) — -//! spec validation, the paragraph/heading/bullet mapping, and the zip pack. -//! That call is **synchronous and CPU-bound by design**: `tinydocs` has no -//! opinion about executors or deadlines, because only a host knows its own. +//! The host keeps the typed contract and validation; OOXML synthesis runs in +//! the loadable document module. This wrapper owns the caller's deadline and +//! maps bus failures onto the agent-facing [`DocumentError`]. //! //! This module supplies exactly that missing policy, and nothing else: //! -//! 1. a `spawn_blocking` hop so a CPU-bound pack never stalls the agent -//! loop's executor, and -//! 2. a `tokio::time::timeout` so a pathological input that slipped past +//! 1. a `tokio::time::timeout` so a pathological input that slipped past //! validation cannot wedge the loop indefinitely, and -//! 3. the mapping from a crate error, a join failure, or an elapsed deadline +//! 2. the mapping from a bus error or elapsed deadline //! onto the agent-facing [`DocumentError`]. //! //! Control flow here is identical to the presentation engine's, so the two @@ -213,7 +209,7 @@ mod tests { // The OOXML round trips that used to live here — container shape, which // text reaches document.xml, blank filtering — moved with the writer into - // `tinydocs::docx`, which tests them against the bytes it produces. Asserting + // the TinyDocs module, which tests them against the bytes it produces. Asserting // them again through a bus call would test the same behaviour twice and // drift the moment one copy changed. diff --git a/src/openhuman/tools/impl/document/format/error/mod.rs b/src/openhuman/tools/impl/document/format/error/mod.rs new file mode 100644 index 0000000000..e46245b59d --- /dev/null +++ b/src/openhuman/tools/impl/document/format/error/mod.rs @@ -0,0 +1,120 @@ +//! Crate-wide error and result types. +//! +//! Every fallible public function in this crate returns [`Result`], and every +//! failure mode is a distinct [`Error`] variant. Add a variant rather than +//! encoding new context into an existing message: callers match on variants, +//! and message text is not a stable API. +//! +//! The variants are deliberately *host-agnostic*. A host that surfaces these +//! to an LLM (the reason [`Error::InvalidInput`] carries a structured +//! `field` / `reason` pair rather than a formatted sentence) maps them onto +//! its own tool-error shape; a host writing to disk maps them onto its own. +//! Nothing here knows about artifacts, timeouts, or async runtimes — those are +//! the host's concerns, because only the host knows its own deadline policy. + +/// Errors returned by this crate. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// A document spec failed validation before any synthesis was attempted. + /// + /// `field` names the offending path in the spec using the same dotted / + /// indexed notation the JSON input uses (`sections[2].bullets[0]`), so an + /// LLM that produced the spec can self-correct without re-reading the + /// whole schema. `reason` states the violated constraint. + #[error("invalid input for field '{field}': {reason}")] + InvalidInput { + /// Path of the offending field within the spec. + field: String, + /// The constraint that was violated. + reason: String, + }, + + /// The underlying document library failed to synthesise the output. + /// + /// `detail` is the library's own error rendered as text and truncated to a + /// bounded length, so the variant never carries an unbounded payload back + /// to a caller that forwards it to a model. + #[error("document generation failed: {detail}")] + GenerationFailed { + /// Truncated underlying library error. + detail: String, + }, + + /// The underlying library failed to extract text from an input document. + /// + /// Distinct from [`Error::GenerationFailed`] because the two have opposite + /// causes and opposite remedies: generation fails on *our* output path and + /// usually means a bug or an exhausted resource, whereas extraction fails on + /// *someone else's* input and usually means the document is damaged, + /// encrypted, or carries no extractable text layer at all. A caller that + /// retries one should not retry the other. + /// + /// `detail` is truncated on the same bound as `GenerationFailed`. + #[error("text extraction failed: {detail}")] + ExtractionFailed { + /// Truncated underlying library error. + detail: String, + }, +} + +impl Error { + /// Maximum length, in Unicode scalar values, of a [`Error::GenerationFailed`] + /// detail string. + pub const MAX_DETAIL_CHARS: usize = 500; + + /// Suffix appended when a detail string is truncated. + const TRUNCATION_SUFFIX: &'static str = " […truncated]"; + + /// Build a [`Error::GenerationFailed`] with `raw` truncated (UTF-8-safe) to + /// [`Error::MAX_DETAIL_CHARS`]. + /// + /// Truncation counts characters, not bytes, so a multi-byte error message + /// can never be cut mid-codepoint. + #[must_use] + pub fn generation_failed(raw: &str) -> Self { + Self::GenerationFailed { + detail: Self::truncate_detail(raw), + } + } + + /// Truncate `raw` to [`Error::MAX_DETAIL_CHARS`] characters, appending the + /// standard truncation suffix when anything was dropped. + #[must_use] + pub fn truncate_detail(raw: &str) -> String { + if raw.chars().count() <= Self::MAX_DETAIL_CHARS { + return raw.to_string(); + } + let keep = Self::MAX_DETAIL_CHARS.saturating_sub(Self::TRUNCATION_SUFFIX.chars().count()); + let mut out: String = raw.chars().take(keep).collect(); + out.push_str(Self::TRUNCATION_SUFFIX); + out + } + + /// Build an [`Error::ExtractionFailed`] with `raw` truncated (UTF-8-safe) to + /// [`Error::MAX_DETAIL_CHARS`]. + #[must_use] + pub fn extraction_failed(raw: &str) -> Self { + Self::ExtractionFailed { + detail: Self::truncate_detail(raw), + } + } + + /// Build an [`Error::InvalidInput`] for `field` violating `reason`. + #[must_use] + pub fn invalid_input(field: impl Into, reason: impl Into) -> Self { + Self::InvalidInput { + field: field.into(), + reason: reason.into(), + } + } +} + +/// The crate's standard result type. +/// +/// Use this alias in public signatures instead of spelling out +/// `std::result::Result`. +pub type Result = std::result::Result; + +#[cfg(test)] +mod test; diff --git a/src/openhuman/tools/impl/document/format/error/test.rs b/src/openhuman/tools/impl/document/format/error/test.rs new file mode 100644 index 0000000000..7bc5815fc1 --- /dev/null +++ b/src/openhuman/tools/impl/document/format/error/test.rs @@ -0,0 +1,54 @@ +//! Unit tests for the crate-wide error type. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::Error; + +#[test] +fn short_details_are_left_intact() { + let err = Error::generation_failed("boom"); + assert_eq!( + err, + Error::GenerationFailed { + detail: "boom".to_string() + } + ); +} + +#[test] +fn long_details_are_truncated_with_a_suffix() { + let raw = "x".repeat(Error::MAX_DETAIL_CHARS * 2); + let Error::GenerationFailed { detail } = Error::generation_failed(&raw) else { + panic!("expected GenerationFailed"); + }; + assert_eq!(detail.chars().count(), Error::MAX_DETAIL_CHARS); + assert!(detail.ends_with("[…truncated]")); +} + +#[test] +fn truncation_never_splits_a_multi_byte_character() { + // Every character is 4 bytes, so a byte-based truncation would panic or + // produce invalid UTF-8. Counting characters keeps the boundary valid. + let raw = "🦀".repeat(Error::MAX_DETAIL_CHARS * 2); + let detail = Error::truncate_detail(&raw); + assert_eq!(detail.chars().count(), Error::MAX_DETAIL_CHARS); + assert!(detail.starts_with('🦀')); +} + +#[test] +fn detail_at_exactly_the_cap_is_not_truncated() { + let raw = "y".repeat(Error::MAX_DETAIL_CHARS); + assert_eq!(Error::truncate_detail(&raw), raw); +} + +#[test] +fn invalid_input_carries_the_field_path_verbatim() { + let err = Error::invalid_input("sections[2].bullets[0]", "must be ≤ 10 chars"); + assert_eq!( + err, + Error::InvalidInput { + field: "sections[2].bullets[0]".to_string(), + reason: "must be ≤ 10 chars".to_string(), + } + ); +} diff --git a/src/openhuman/tools/impl/document/format/mod.rs b/src/openhuman/tools/impl/document/format/mod.rs new file mode 100644 index 0000000000..81c45df4d8 --- /dev/null +++ b/src/openhuman/tools/impl/document/format/mod.rs @@ -0,0 +1,29 @@ +//! Agent-friendly document synthesis and text extraction in Rust. +//! +//! Typed, validated document contracts shared with the document bus module. +//! They are built for hosts that let a language model produce documents: +//! the spec types are the JSON tool schema, validation rejects a malformed +//! spec with a structured [`Error::InvalidInput`] naming the exact field so +//! the model can self-correct, and synthesis returns a plain byte buffer. +//! +//! # What this module deliberately does not do +//! +//! No filesystem access, no subprocesses, no async runtime, no deadline +//! handling. Synthesis runs in the document bus module; this host module owns +//! the wire contract and validation only. +//! +//! # Layout +//! +//! - [`error`](self::Error) — the crate-wide [`Error`] and [`Result`]. +//! - [`spec`] — the typed document specs and their validation. Compiled in +//! every build, including `--no-default-features`, so a host whose synthesis +//! happens elsewhere still shares one definition of the wire contract. +//! +//! Writer and extractor implementations are intentionally absent: the host +//! sends these contract values over TinyBus. + +mod error; + +pub mod spec; + +pub use error::{Error, Result}; diff --git a/src/openhuman/tools/impl/document/format/spec/document/mod.rs b/src/openhuman/tools/impl/document/format/spec/document/mod.rs new file mode 100644 index 0000000000..d96e78f429 --- /dev/null +++ b/src/openhuman/tools/impl/document/format/spec/document/mod.rs @@ -0,0 +1,261 @@ +//! The `.docx` document spec: the typed description a caller hands to +//! `docx::generate`, plus the size limits every spec is validated against. +//! +//! The spec is the crate's wire contract. It derives `Serialize` / +//! `Deserialize` with `deny_unknown_fields` because the usual caller is an +//! LLM tool boundary: the same struct that drives synthesis is the one whose +//! JSON schema the model is shown, and a typo'd field name should be a loud +//! rejection rather than a silently ignored key. +//! +//! Limits are public consts rather than private constants so a host can quote +//! the exact number in its own tool description and stay in lockstep with what +//! validation actually enforces. +//! +//! Nothing in this module depends on the `docx` feature or on `docx-rs`: it is +//! `serde` plus the crate error type. A host that only needs to *describe* and +//! *validate* a document — because synthesis happens elsewhere, in another +//! process or behind a message bus — can therefore depend on this crate with +//! `default-features = false` and still share one definition of the contract. + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::tools::implementations::document::format::{Error, Result}; + +/// Maximum number of sections a single document may contain. +/// +/// Bounds generation time and output size; a caller with more material is +/// expected to split it across multiple documents. +pub const MAX_SECTIONS: usize = 128; + +/// Maximum length, in Unicode scalar values, of a short text field — the +/// document title, the author byline, or a section heading. +pub const MAX_TEXT_CHARS: usize = 2_000; + +/// Maximum length, in Unicode scalar values, of a single body paragraph or +/// bullet item. +/// +/// More generous than [`MAX_TEXT_CHARS`]: prose paragraphs legitimately run +/// far longer than a heading. +pub const MAX_PARAGRAPH_CHARS: usize = 20_000; + +/// Maximum number of body paragraphs in a single section. +pub const MAX_PARAGRAPHS_PER_SECTION: usize = 200; + +/// Maximum number of bullet-list items in a single section. +pub const MAX_BULLETS_PER_SECTION: usize = 200; + +/// Aggregate cap on all renderable text across the whole document — the +/// title, the author byline, and every section's heading, paragraphs, and +/// bullets — in Unicode scalar values. +/// +/// The per-field and per-section limits above bound each individual piece, but +/// not their product — `MAX_SECTIONS × MAX_PARAGRAPHS_PER_SECTION × +/// MAX_PARAGRAPH_CHARS` alone is over 500M characters, so a spec satisfying +/// every other limit could still build a multi-hundred-megabyte document in +/// memory. This total keeps the worst case bounded to a few megabytes of text +/// while staying generous for any real document. +pub const MAX_TOTAL_CHARS: usize = 2_000_000; + +/// One section of the document, rendered in spec order. +/// +/// A section is an optional heading followed by any number of body paragraphs +/// and/or a bullet list. At least one of the three must carry renderable text — +/// a wholly blank section is rejected by [`DocumentSpec::validate`] rather than +/// silently rendering nothing. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DocumentSection { + /// Section heading, rendered as a bold heading paragraph. Optional: a + /// section may be pure body text under the document title. + #[serde(default)] + pub heading: Option, + /// Body paragraphs, each rendered as its own paragraph, in order. + /// Blank and whitespace-only entries are dropped during synthesis. + #[serde(default)] + pub paragraphs: Vec, + /// Bullet-list items, rendered as a single-level bulleted list after the + /// section's body paragraphs. Blank and whitespace-only entries are + /// dropped during synthesis. + #[serde(default)] + pub bullets: Vec, +} + +impl DocumentSection { + /// Returns `true` when the section carries no renderable content at all — + /// the heading is absent or blank, and every paragraph and bullet is blank. + /// + /// Synthesis trims and drops blank entries, so a section holding only + /// `[" "]` would render as nothing despite carrying entries. Validation + /// uses this to reject that case up front. + #[must_use] + pub fn is_blank(&self) -> bool { + let has_heading = self + .heading + .as_deref() + .is_some_and(|h| !h.trim().is_empty()); + let has_paragraph = self.paragraphs.iter().any(|p| !p.trim().is_empty()); + let has_bullet = self.bullets.iter().any(|b| !b.trim().is_empty()); + !(has_heading || has_paragraph || has_bullet) + } +} + +/// A complete `.docx` document spec. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DocumentSpec { + /// Document title, rendered as the leading title paragraph. Required and + /// non-blank. + pub title: String, + /// Optional author byline, rendered as an italic line beneath the title. + #[serde(default)] + pub author: Option, + /// Sections, in display order. Must contain at least one entry. + #[serde(default)] + pub sections: Vec, +} + +impl DocumentSpec { + /// Total renderable text across the whole spec, in Unicode scalar values. + /// + /// Sums with saturating arithmetic so an adversarial spec cannot overflow + /// the counter into a small value that passes the aggregate check. + #[must_use] + pub fn total_chars(&self) -> usize { + let mut total = self.title.chars().count(); + if let Some(author) = self.author.as_deref() { + total = total.saturating_add(author.chars().count()); + } + for section in &self.sections { + if let Some(heading) = section.heading.as_deref() { + total = total.saturating_add(heading.chars().count()); + } + for paragraph in §ion.paragraphs { + total = total.saturating_add(paragraph.chars().count()); + } + for bullet in §ion.bullets { + total = total.saturating_add(bullet.chars().count()); + } + } + total + } + + /// Check the spec against every documented size limit. + /// + /// Callers do not have to invoke this: `docx::generate` validates before it + /// synthesises anything. It is public so a host can reject a malformed + /// spec at its own boundary — an LLM tool call, say — and hand back the + /// structured [`Error::InvalidInput`] before paying for a blocking hop, a + /// process boundary, or a bus round trip. + /// + /// # Errors + /// + /// Returns [`Error::InvalidInput`] naming the first field that violates a + /// limit. Fields are checked in spec order (title, author, sections, then + /// each section's contents) so the reported field is stable for a given + /// spec. + pub fn validate(&self) -> Result<()> { + if self.title.trim().is_empty() { + return Err(Error::invalid_input("title", "must not be empty")); + } + if self.title.chars().count() > MAX_TEXT_CHARS { + return Err(Error::invalid_input( + "title", + format!("must be ≤ {MAX_TEXT_CHARS} chars"), + )); + } + // Running total across every renderable field — title, author, and all + // section contents — checked as each field is processed. A spec can pass + // every per-field limit yet blow the aggregate budget, and checking + // incrementally rejects it as soon as the budget is crossed without a + // second pass over the whole spec. + let over_budget = || { + Error::invalid_input( + "sections", + format!("total document text must be ≤ {MAX_TOTAL_CHARS} chars"), + ) + }; + let mut total = self.title.chars().count(); + if let Some(author) = self.author.as_deref() { + if author.chars().count() > MAX_TEXT_CHARS { + return Err(Error::invalid_input( + "author", + format!("must be ≤ {MAX_TEXT_CHARS} chars"), + )); + } + total = total.saturating_add(author.chars().count()); + } + if self.sections.is_empty() { + return Err(Error::invalid_input( + "sections", + "must contain at least one section", + )); + } + if self.sections.len() > MAX_SECTIONS { + return Err(Error::invalid_input( + "sections", + format!("must contain ≤ {MAX_SECTIONS} sections"), + )); + } + + for (i, section) in self.sections.iter().enumerate() { + if section.is_blank() { + return Err(Error::invalid_input( + format!("sections[{i}]"), + "must have at least one of heading / paragraphs / bullets", + )); + } + if let Some(heading) = section.heading.as_deref() { + if heading.chars().count() > MAX_TEXT_CHARS { + return Err(Error::invalid_input( + format!("sections[{i}].heading"), + format!("must be ≤ {MAX_TEXT_CHARS} chars"), + )); + } + total = total.saturating_add(heading.chars().count()); + if total > MAX_TOTAL_CHARS { + return Err(over_budget()); + } + } + if section.paragraphs.len() > MAX_PARAGRAPHS_PER_SECTION { + return Err(Error::invalid_input( + format!("sections[{i}].paragraphs"), + format!("must contain ≤ {MAX_PARAGRAPHS_PER_SECTION} paragraphs"), + )); + } + for (p, paragraph) in section.paragraphs.iter().enumerate() { + if paragraph.chars().count() > MAX_PARAGRAPH_CHARS { + return Err(Error::invalid_input( + format!("sections[{i}].paragraphs[{p}]"), + format!("must be ≤ {MAX_PARAGRAPH_CHARS} chars"), + )); + } + total = total.saturating_add(paragraph.chars().count()); + if total > MAX_TOTAL_CHARS { + return Err(over_budget()); + } + } + if section.bullets.len() > MAX_BULLETS_PER_SECTION { + return Err(Error::invalid_input( + format!("sections[{i}].bullets"), + format!("must contain ≤ {MAX_BULLETS_PER_SECTION} bullets"), + )); + } + for (b, bullet) in section.bullets.iter().enumerate() { + if bullet.chars().count() > MAX_PARAGRAPH_CHARS { + return Err(Error::invalid_input( + format!("sections[{i}].bullets[{b}]"), + format!("must be ≤ {MAX_PARAGRAPH_CHARS} chars"), + )); + } + total = total.saturating_add(bullet.chars().count()); + if total > MAX_TOTAL_CHARS { + return Err(over_budget()); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/tools/impl/document/format/spec/document/test.rs b/src/openhuman/tools/impl/document/format/spec/document/test.rs new file mode 100644 index 0000000000..724302217e --- /dev/null +++ b/src/openhuman/tools/impl/document/format/spec/document/test.rs @@ -0,0 +1,272 @@ +//! Unit tests for the wire contracts: validation, the blank/aggregate rules, +//! and JSON round-tripping. +//! +//! These are deliberately separate from the format modules' tests. They must +//! pass in a build with every format feature off, because the spec is the half +//! of the crate a bus- or process-boundary host shares without the codec. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{ + DocumentSection, DocumentSpec, MAX_BULLETS_PER_SECTION, MAX_PARAGRAPHS_PER_SECTION, + MAX_PARAGRAPH_CHARS, MAX_SECTIONS, MAX_TEXT_CHARS, MAX_TOTAL_CHARS, +}; +use crate::openhuman::tools::implementations::document::format::Error; + +/// One valid section carrying a heading, a paragraph, and a bullet. +fn section() -> DocumentSection { + DocumentSection { + heading: Some("Overview".to_string()), + paragraphs: vec!["A body paragraph.".to_string()], + bullets: vec!["A bullet".to_string()], + } +} + +/// A minimal valid spec; each test mutates one field to drive a single branch. +fn spec() -> DocumentSpec { + DocumentSpec { + title: "Charter".to_string(), + author: Some("Alice".to_string()), + sections: vec![section()], + } +} + +/// Assert `spec` is rejected with an `InvalidInput` naming `field`. +fn assert_rejects(spec: &DocumentSpec, field: &str) { + match spec.validate() { + Err(Error::InvalidInput { field: f, .. }) => { + assert_eq!(f, field, "unexpected rejected field"); + } + other => panic!("expected InvalidInput({field}), got {other:?}"), + } +} + +#[test] +fn accepts_a_well_formed_spec() { + assert!(spec().validate().is_ok()); +} + +#[test] +fn rejects_a_blank_title() { + let mut s = spec(); + s.title = " ".to_string(); + assert_rejects(&s, "title"); +} + +#[test] +fn rejects_an_over_long_title() { + let mut s = spec(); + s.title = "t".repeat(MAX_TEXT_CHARS + 1); + assert_rejects(&s, "title"); +} + +#[test] +fn rejects_an_over_long_author() { + let mut s = spec(); + s.author = Some("a".repeat(MAX_TEXT_CHARS + 1)); + assert_rejects(&s, "author"); +} + +#[test] +fn rejects_a_spec_with_no_sections() { + let mut s = spec(); + s.sections.clear(); + assert_rejects(&s, "sections"); +} + +#[test] +fn rejects_too_many_sections() { + let mut s = spec(); + s.sections = vec![section(); MAX_SECTIONS + 1]; + assert_rejects(&s, "sections"); +} + +#[test] +fn rejects_a_wholly_blank_section() { + // Every entry is present but whitespace-only, so synthesis would drop all + // of them and render nothing. Validation catches it instead. + let mut s = spec(); + s.sections = vec![DocumentSection { + heading: Some(" ".to_string()), + paragraphs: vec!["\t".to_string()], + bullets: vec![String::new()], + }]; + assert_rejects(&s, "sections[0]"); +} + +#[test] +fn rejects_an_over_long_heading_naming_its_index() { + let mut s = spec(); + s.sections.push(DocumentSection { + heading: Some("h".repeat(MAX_TEXT_CHARS + 1)), + ..section() + }); + assert_rejects(&s, "sections[1].heading"); +} + +#[test] +fn rejects_too_many_paragraphs() { + let mut s = spec(); + s.sections[0].paragraphs = vec!["p".to_string(); MAX_PARAGRAPHS_PER_SECTION + 1]; + assert_rejects(&s, "sections[0].paragraphs"); +} + +#[test] +fn rejects_an_over_long_paragraph_naming_its_index() { + let mut s = spec(); + s.sections[0].paragraphs = vec!["ok".to_string(), "p".repeat(MAX_PARAGRAPH_CHARS + 1)]; + assert_rejects(&s, "sections[0].paragraphs[1]"); +} + +#[test] +fn rejects_too_many_bullets() { + let mut s = spec(); + s.sections[0].bullets = vec!["b".to_string(); MAX_BULLETS_PER_SECTION + 1]; + assert_rejects(&s, "sections[0].bullets"); +} + +#[test] +fn rejects_an_over_long_bullet_naming_its_index() { + let mut s = spec(); + s.sections[0].bullets = vec!["ok".to_string(), "b".repeat(MAX_PARAGRAPH_CHARS + 1)]; + assert_rejects(&s, "sections[0].bullets[1]"); +} + +#[test] +fn rejects_a_spec_over_the_aggregate_character_budget() { + // Each individual field is within its own limit; only the sum is not. One + // section with just enough max-length paragraphs to cross MAX_TOTAL_CHARS + // reproduces that without allocating hundreds of megabytes: repeating a + // whole section MAX_SECTIONS times (the original fixture) built ~512 MB + // of paragraph text before validation ever ran. + let paragraph_count = MAX_TOTAL_CHARS / MAX_PARAGRAPH_CHARS + 1; + assert!(paragraph_count <= MAX_PARAGRAPHS_PER_SECTION); + let paragraph = "x".repeat(MAX_PARAGRAPH_CHARS); + let big = DocumentSection { + heading: Some("Heading".to_string()), + paragraphs: vec![paragraph; paragraph_count], + bullets: vec![], + }; + let s = DocumentSpec { + title: "Huge".to_string(), + author: None, + sections: vec![big], + }; + // Sanity: this spec passes every per-field check. + assert!(s.sections.len() <= MAX_SECTIONS); + assert_rejects(&s, "sections"); +} + +#[test] +fn rejects_an_aggregate_overrun_that_a_bullet_crosses() { + // The heading and paragraph loops each carry their own budget check; so does + // the bullet loop, and only a spec whose overrun lands on a bullet drives + // that third branch. + let bullet = "b".repeat(MAX_PARAGRAPH_CHARS); + let bullet_count = MAX_TOTAL_CHARS / MAX_PARAGRAPH_CHARS + 1; + assert!(bullet_count <= MAX_BULLETS_PER_SECTION); + let s = DocumentSpec { + title: "Bullets".to_string(), + author: None, + sections: vec![DocumentSection { + heading: None, + paragraphs: vec![], + bullets: vec![bullet; bullet_count], + }], + }; + assert_rejects(&s, "sections"); +} + +#[test] +fn rejects_an_aggregate_overrun_that_a_heading_crosses() { + // Headings cannot reach the aggregate cap on their own: MAX_SECTIONS × + // MAX_TEXT_CHARS is 256_000, two orders of magnitude under MAX_TOTAL_CHARS. + // Driving the heading branch therefore means spending the budget down to a + // single character of headroom in an earlier section, then letting a + // perfectly legal heading cross it. + let title = "Headings"; + let filler_count = MAX_TOTAL_CHARS / MAX_PARAGRAPH_CHARS - 1; + assert!(filler_count <= MAX_PARAGRAPHS_PER_SECTION); + let used = title.chars().count() + filler_count * MAX_PARAGRAPH_CHARS; + // Leave exactly one character of headroom. + let tail = MAX_TOTAL_CHARS - used - 1; + assert!(tail <= MAX_PARAGRAPH_CHARS); + + let mut paragraphs = vec!["p".repeat(MAX_PARAGRAPH_CHARS); filler_count]; + paragraphs.push("p".repeat(tail)); + + let s = DocumentSpec { + title: title.to_string(), + author: None, + sections: vec![ + DocumentSection { + heading: None, + paragraphs, + bullets: vec![], + }, + DocumentSection { + // Two characters against one character of headroom. + heading: Some("hh".to_string()), + paragraphs: vec![], + bullets: vec![], + }, + ], + }; + assert!(s.sections.len() <= MAX_SECTIONS); + assert_rejects(&s, "sections"); +} + +#[test] +fn is_blank_reflects_content_presence() { + assert!(!section().is_blank()); + assert!(DocumentSection { + heading: None, + paragraphs: vec![], + bullets: vec![], + } + .is_blank()); + // A heading alone is enough content. + assert!(!DocumentSection { + heading: Some("Only a heading".to_string()), + paragraphs: vec![], + bullets: vec![], + } + .is_blank()); +} + +#[test] +fn total_chars_sums_every_text_field() { + let s = DocumentSpec { + title: "abcd".to_string(), // 4 + author: Some("xy".to_string()), // 2 + sections: vec![DocumentSection { + heading: Some("hij".to_string()), // 3 + paragraphs: vec!["pq".to_string()], // 2 + bullets: vec!["b".to_string()], // 1 + }], + }; + assert_eq!(s.total_chars(), 12); +} + +#[test] +fn spec_round_trips_through_json() { + let s = spec(); + let json = serde_json::to_string(&s).expect("serialises"); + let back: DocumentSpec = serde_json::from_str(&json).expect("deserialises"); + assert_eq!(back, s); +} + +#[test] +fn spec_rejects_unknown_json_fields() { + // `deny_unknown_fields` makes a typo'd key a loud rejection rather than a + // silently ignored one — the whole point at an LLM tool boundary. + let json = r#"{"title":"T","sections":[],"titel":"typo"}"#; + assert!(serde_json::from_str::(json).is_err()); +} + +#[test] +fn spec_defaults_optional_fields() { + let s: DocumentSpec = serde_json::from_str(r#"{"title":"T"}"#).expect("deserialises"); + assert_eq!(s.author, None); + assert!(s.sections.is_empty()); +} diff --git a/src/openhuman/tools/impl/document/format/spec/image/mod.rs b/src/openhuman/tools/impl/document/format/spec/image/mod.rs new file mode 100644 index 0000000000..4fab9d934e --- /dev/null +++ b/src/openhuman/tools/impl/document/format/spec/image/mod.rs @@ -0,0 +1,161 @@ +//! Raster-image identification for specs that embed images. +//! +//! Two formats are supported, PNG and JPEG, and the restriction is deliberate +//! rather than incidental: the OOXML presentation writer this crate drives +//! declares no `webp` default in the generated `[Content_Types].xml`, and its +//! automatic format detection misclassifies `webp` as PNG — producing a part +//! `PowerPoint` refuses to render. Accepting only what can actually be embedded +//! turns that into a clean rejection at the boundary. +//! +//! Identification is done by reading the container header directly, in about a +//! hundred lines and with no dependencies, rather than by pulling in a decoding +//! stack. Nothing here decodes pixels: it answers "which format is this" and +//! "what are its native dimensions", which is all a layout engine needs to +//! place an image with the right aspect ratio. +//! +//! Like the rest of [`crate::openhuman::tools::implementations::document::format::spec`], this module is compiled in every build. A +//! host resolving image bytes has to identify and measure them to *build* a +//! spec, and that must not require the writer. + +use serde::{Deserialize, Serialize}; + +/// A raster image format that can be embedded in a generated document. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum ImageFormat { + /// Portable Network Graphics. + Png, + /// JPEG / JFIF. + Jpeg, +} + +impl ImageFormat { + /// The format's canonical OOXML name — `"PNG"` or `"JPEG"`. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Png => "PNG", + Self::Jpeg => "JPEG", + } + } + + /// Identify `bytes` by its container header. + /// + /// Returns `None` for a truncated header or any format other than the two + /// embeddable ones — including GIF, WebP and BMP, which are recognisable + /// but not embeddable. + #[must_use] + pub fn sniff(bytes: &[u8]) -> Option { + if bytes.starts_with(&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]) { + Some(Self::Png) + } else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + Some(Self::Jpeg) + } else { + None + } + } + + /// Native `(width, height)` of `bytes` in pixels, read from the header. + /// + /// Returns `None` when the header is truncated or malformed, or when either + /// dimension is zero — a degenerate image cannot be placed aspect-correctly + /// and is rejected rather than divided by. + #[must_use] + pub fn dimensions(self, bytes: &[u8]) -> Option<(u32, u32)> { + match self { + Self::Png => png_dimensions(bytes), + Self::Jpeg => jpeg_dimensions(bytes), + } + } +} + +impl std::fmt::Display for ImageFormat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// PNG: 8-byte signature, then an `IHDR` chunk whose width / height are +/// big-endian `u32`s at byte offsets 16 and 20. +fn png_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + if bytes.len() < 24 || &bytes[12..16] != b"IHDR" { + return None; + } + let w = u32::from_be_bytes([bytes[16], bytes[17], bytes[18], bytes[19]]); + let h = u32::from_be_bytes([bytes[20], bytes[21], bytes[22], bytes[23]]); + if w == 0 || h == 0 { + return None; + } + Some((w, h)) +} + +/// JPEG: walk the marker segments until a Start-Of-Frame is hit; its payload +/// carries height then width as big-endian `u16`s. +fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + let mut i = 2; // skip the leading FF D8 SOI + while i + 3 < bytes.len() { + if bytes[i] != 0xFF { + i += 1; + continue; + } + let marker = bytes[i + 1]; + i += 2; + // Standalone markers carry no length field: padding fill bytes, TEM, + // RSTn, SOI and EOI. Reading the next two bytes as a length here would + // desynchronise the walk and reject a valid file — TEM in particular is + // legal before the frame header. + if marker == 0xFF + || marker == 0x01 + || marker == 0xD8 + || marker == 0xD9 + || (0xD0..=0xD7).contains(&marker) + { + continue; + } + if i + 1 >= bytes.len() { + return None; + } + let seg_len = u16::from_be_bytes([bytes[i], bytes[i + 1]]) as usize; + if seg_len < 2 { + return None; + } + // SOF markers carrying frame dimensions. Excludes 0xC4 (DHT), + // 0xC8 (JPG) and 0xCC (DAC), which share the 0xCn range but are not + // frame headers. + let is_sof = matches!( + marker, + 0xC0 | 0xC1 + | 0xC2 + | 0xC3 + | 0xC5 + | 0xC6 + | 0xC7 + | 0xC9 + | 0xCA + | 0xCB + | 0xCD + | 0xCE + | 0xCF + ); + if is_sof { + // segment: [len_hi len_lo precision h_hi h_lo w_hi w_lo ...] + if i + 6 >= bytes.len() { + return None; + } + let h = u32::from(u16::from_be_bytes([bytes[i + 3], bytes[i + 4]])); + let w = u32::from(u16::from_be_bytes([bytes[i + 5], bytes[i + 6]])); + if w == 0 || h == 0 { + return None; + } + return Some((w, h)); + } + i += seg_len; + } + None +} + +// Visible crate-wide under `cfg(test)`: the `png` / `jpeg` header builders here +// are the fixtures every image-carrying spec and every synthesis test needs, and +// one honest builder beats a base64 blob copied into three files. +#[cfg(test)] +pub(crate) mod test; diff --git a/src/openhuman/tools/impl/document/format/spec/image/test.rs b/src/openhuman/tools/impl/document/format/spec/image/test.rs new file mode 100644 index 0000000000..90206fce8a --- /dev/null +++ b/src/openhuman/tools/impl/document/format/spec/image/test.rs @@ -0,0 +1,151 @@ +//! Unit tests for image identification and header measurement. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{jpeg_dimensions, png_dimensions, ImageFormat}; + +/// A 1×1 PNG assembled byte-for-byte: signature, `IHDR`, `IDAT`, `IEND`. +/// +/// Built literally rather than decoded from base64 so the fixture needs no +/// dependency and the offsets under test are visible in the source. +pub(crate) fn png(width: u32, height: u32) -> Vec { + let mut out = vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + out.extend_from_slice(&13u32.to_be_bytes()); // IHDR length + out.extend_from_slice(b"IHDR"); + out.extend_from_slice(&width.to_be_bytes()); + out.extend_from_slice(&height.to_be_bytes()); + out.extend_from_slice(&[0x08, 0x06, 0x00, 0x00, 0x00]); // depth, colour, etc. + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); // CRC placeholder + out.extend_from_slice(&0u32.to_be_bytes()); // empty IDAT + out.extend_from_slice(b"IDAT"); + out.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); + out.extend_from_slice(&0u32.to_be_bytes()); + out.extend_from_slice(b"IEND"); + out.extend_from_slice(&[0xAE, 0x42, 0x60, 0x82]); + out +} + +/// A minimal JPEG: SOI, an APP0 stub, then an SOF0 declaring `height × width`. +pub(crate) fn jpeg(width: u16, height: u16) -> Vec { + let mut out = vec![ + 0xFF, 0xD8, // SOI + 0xFF, 0xE0, 0x00, 0x04, 0x00, 0x00, // APP0, len=4, 2 payload bytes + 0xFF, 0xC0, 0x00, 0x0B, // SOF0, len=11 + 0x08, // precision + ]; + out.extend_from_slice(&height.to_be_bytes()); + out.extend_from_slice(&width.to_be_bytes()); + out.extend_from_slice(&[0x03, 0x00, 0x00, 0x00]); // components (filler) + out.extend_from_slice(&[0xFF, 0xD9]); // EOI + out +} + +#[test] +fn sniffs_png_and_jpeg() { + assert_eq!(ImageFormat::sniff(&png(1, 1)), Some(ImageFormat::Png)); + assert_eq!(ImageFormat::sniff(&jpeg(7, 5)), Some(ImageFormat::Jpeg)); +} + +#[test] +fn rejects_non_images_and_unembeddable_formats() { + assert_eq!(ImageFormat::sniff(b"not an image"), None); + // GIF and WebP are recognisable, but the writer cannot embed either. + assert_eq!(ImageFormat::sniff(b"GIF89a....."), None); + assert_eq!(ImageFormat::sniff(b"RIFF\0\0\0\0WEBP"), None); + assert_eq!(ImageFormat::sniff(&[]), None); +} + +#[test] +fn reads_png_dimensions() { + assert_eq!(ImageFormat::Png.dimensions(&png(1, 1)), Some((1, 1)), "1x1"); + assert_eq!( + ImageFormat::Png.dimensions(&png(1920, 1080)), + Some((1920, 1080)) + ); +} + +#[test] +fn reads_jpeg_dimensions() { + assert_eq!(ImageFormat::Jpeg.dimensions(&jpeg(7, 5)), Some((7, 5))); +} + +#[test] +fn truncated_headers_yield_none() { + assert_eq!(png_dimensions(&[0x89, 0x50, 0x4E, 0x47]), None); + assert_eq!(jpeg_dimensions(&[0xFF, 0xD8]), None); +} + +#[test] +fn a_png_without_an_ihdr_chunk_yields_none() { + let mut bytes = png(4, 4); + bytes[12..16].copy_from_slice(b"XXXX"); + assert_eq!(png_dimensions(&bytes), None); +} + +#[test] +fn a_zero_dimension_yields_none() { + // Degenerate images cannot be placed aspect-correctly; they are rejected + // rather than divided by. + assert_eq!(png_dimensions(&png(0, 8)), None); + assert_eq!(png_dimensions(&png(8, 0)), None); + assert_eq!(jpeg_dimensions(&jpeg(0, 8)), None); + assert_eq!(jpeg_dimensions(&jpeg(8, 0)), None); +} + +#[test] +fn a_jpeg_with_no_start_of_frame_yields_none() { + // SOI, then an APP0 segment and EOI — a valid marker stream carrying no + // frame header at all. + let bytes = vec![ + 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x04, 0x00, 0x00, 0xFF, 0xD9, 0x00, 0x00, + ]; + assert_eq!(jpeg_dimensions(&bytes), None); +} + +#[test] +fn a_jpeg_with_a_degenerate_segment_length_yields_none() { + // A declared segment length below the two length bytes themselves would + // make the walk loop forever if it were trusted. + let bytes = vec![0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x01, 0x00, 0x00, 0x00]; + assert_eq!(jpeg_dimensions(&bytes), None); +} + +#[test] +fn a_jpeg_skips_standalone_and_non_frame_markers_before_the_frame() { + // Restart markers and a DHT (0xC4, in the 0xCn range but not a frame + // header) must both be stepped over rather than mistaken for an SOF. + let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xD0, 0xFF, 0xFF]; + bytes.extend_from_slice(&[0xFF, 0xC4, 0x00, 0x04, 0x00, 0x00]); // DHT + bytes.extend_from_slice(&[0xFF, 0xC0, 0x00, 0x0B, 0x08]); + bytes.extend_from_slice(&11u16.to_be_bytes()); // height + bytes.extend_from_slice(&22u16.to_be_bytes()); // width + bytes.extend_from_slice(&[0x03, 0x00, 0x00, 0x00, 0xFF, 0xD9]); + assert_eq!(jpeg_dimensions(&bytes), Some((22, 11))); +} + +#[test] +fn a_jpeg_with_a_tem_marker_before_the_frame_is_still_measured() { + // TEM (0xFF01) carries no length field. Reading the next two bytes as one + // desynchronises the walk and rejects a valid file. + let mut bytes = vec![0xFF, 0xD8, 0xFF, 0x01]; + bytes.extend_from_slice(&[0xFF, 0xC0, 0x00, 0x0B, 0x08]); + bytes.extend_from_slice(&33u16.to_be_bytes()); // height + bytes.extend_from_slice(&44u16.to_be_bytes()); // width + bytes.extend_from_slice(&[0x03, 0x00, 0x00, 0x00, 0xFF, 0xD9]); + assert_eq!(jpeg_dimensions(&bytes), Some((44, 33))); +} + +#[test] +fn format_renders_its_ooxml_name() { + assert_eq!(ImageFormat::Png.as_str(), "PNG"); + assert_eq!(ImageFormat::Jpeg.as_str(), "JPEG"); + assert_eq!(ImageFormat::Jpeg.to_string(), "JPEG"); +} + +#[test] +fn format_round_trips_through_json_as_its_ooxml_name() { + let json = serde_json::to_string(&ImageFormat::Png).expect("serialises"); + assert_eq!(json, r#""PNG""#); + let back: ImageFormat = serde_json::from_str(&json).expect("deserialises"); + assert_eq!(back, ImageFormat::Png); +} diff --git a/src/openhuman/tools/impl/document/format/spec/mod.rs b/src/openhuman/tools/impl/document/format/spec/mod.rs new file mode 100644 index 0000000000..21f8a2eaeb --- /dev/null +++ b/src/openhuman/tools/impl/document/format/spec/mod.rs @@ -0,0 +1,45 @@ +//! The wire contracts: typed document specs and their validation, with no +//! dependency on any format writer. +//! +//! Every format module in this crate (`docx`, `pptx`, …) synthesises bytes from +//! a spec defined here. The split matters for two reasons: +//! +//! 1. **A host can share the contract without paying for the codec.** This +//! module is `serde` plus the crate [`Error`](crate::openhuman::tools::implementations::document::format::Error) — nothing else. +//! It is compiled in *every* build, including `--no-default-features`, so a +//! host whose synthesis happens elsewhere (in another process, or behind a +//! message bus) still gets the one authoritative definition of the spec +//! instead of re-declaring it and drifting. +//! 2. **Validation is cheap and belongs at the boundary.** The specs validate +//! themselves without touching a writer, so a host can reject a malformed +//! LLM tool call before paying for a blocking hop or a round trip. +//! +//! # Where things live +//! +//! - [`document`] — `.docx`: [`DocumentSpec`], [`DocumentSection`]. +//! - [`presentation`] — `.pptx`: [`PresentationSpec`], [`SlideSpec`], +//! [`SlideImage`]. +//! - [`image`] — [`ImageFormat`], for specs that embed raster images. +//! +//! **Types are re-exported here; limits are not.** Each format's limits stay +//! inside its own module, because the same name means a different thing in each +//! — `document::MAX_TEXT_CHARS` bounds a heading, `presentation::MAX_TEXT_CHARS` +//! bounds a bullet — and flattening them would put two distinct constants under +//! one name. Reach for `spec::presentation::MAX_SLIDES` and read it as the +//! sentence it is. +//! +//! The format modules re-export both the types and the limits they consume, so +//! `crate::openhuman::tools::implementations::document::format::docx::DocumentSpec` and [`crate::openhuman::tools::implementations::document::format::spec::DocumentSpec`] name the +//! same type. +//! +//! [`crate::openhuman::tools::implementations::document::format::spec::DocumentSpec`]: DocumentSpec + +pub mod document; +pub mod image; +pub mod presentation; + +pub use document::{DocumentSection, DocumentSpec}; +pub use image::ImageFormat; +pub use presentation::wire::{WirePresentationSpec, WireSlideImage, WireSlideSpec}; +#[allow(unused_imports)] +pub use presentation::{PresentationSpec, SlideImage, SlideSpec}; diff --git a/src/openhuman/tools/impl/document/format/spec/presentation/mod.rs b/src/openhuman/tools/impl/document/format/spec/presentation/mod.rs new file mode 100644 index 0000000000..9e030540cc --- /dev/null +++ b/src/openhuman/tools/impl/document/format/spec/presentation/mod.rs @@ -0,0 +1,340 @@ +//! The `.pptx` presentation spec: the typed description a caller hands to +//! `pptx::generate`, plus the size limits every spec is validated against. +//! +//! Same contract rules as [`crate::openhuman::tools::implementations::document::format::spec::document`] — `deny_unknown_fields`, +//! public limits, `validate` before synthesis — with one structural difference +//! worth understanding. +//! +//! # Images are bytes here, not references +//! +//! A [`SlideImage`] carries the image *bytes*, its format, and its native pixel +//! dimensions. It deliberately does **not** carry a path, a URL, or an +//! application-specific identifier, because resolving any of those is host +//! policy this crate has no business holding: which directories an agent may +//! read, whether a given identifier belongs to the caller, and whether fetching +//! a URL is an acceptable request to originate are all questions with different +//! answers in every host. A host resolves indirection under its own rules and +//! hands over the resulting bytes. +//! +//! [`SlideImage::from_bytes`] does the mechanical half of that hand-off: +//! identify the format and read the dimensions, or reject the bytes. It needs +//! no format writer, so a host can build and validate a whole spec in a build +//! with the `pptx` feature off. + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::tools::implementations::document::format::spec::image::ImageFormat; +use crate::openhuman::tools::implementations::document::format::{Error, Result}; + +/// Maximum number of content slides a single deck may contain. +/// +/// Bounds generation time and output size; a caller with more material is +/// expected to split it across multiple decks. +pub const MAX_SLIDES: usize = 64; + +/// Maximum length, in Unicode scalar values, of any single text field — the +/// deck title, the author byline, the theme hint, a slide title, a slide body, +/// one bullet, the speaker notes, or an image caption. +pub const MAX_TEXT_CHARS: usize = 2_000; + +/// Maximum number of bullets on a single slide. +/// +/// Higher counts produce a slide nobody can read, and bloat the output. +pub const MAX_BULLETS_PER_SLIDE: usize = 32; + +/// Maximum number of images attached to a single slide. +/// +/// The single-column layout stacks images vertically in the lower band of the +/// slide; past this count each one is too small to read. +pub const MAX_IMAGES_PER_SLIDE: usize = 6; + +/// Maximum number of images across the whole deck. +/// +/// Bounds the embedded media payload regardless of how the images are +/// distributed across slides. +pub const MAX_IMAGES_PER_DECK: usize = 8; + +/// Maximum size, in bytes, of a single embedded image. +pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024; + +/// One image embedded on a slide. +/// +/// Construct with [`SlideImage::from_bytes`] rather than by hand: it derives +/// `format` and the dimensions from the bytes, which keeps the three fields +/// consistent by construction. [`PresentationSpec::validate`] re-checks that +/// consistency, because a spec can also arrive over a wire. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SlideImage { + /// The encoded image, as PNG or JPEG bytes. + pub bytes: Vec, + /// The format of `bytes`. + pub format: ImageFormat, + /// Native width in pixels, used to place the image without distorting it. + pub width_px: u32, + /// Native height in pixels, used to place the image without distorting it. + pub height_px: u32, + /// Optional caption, rendered as a bullet beneath the image. + #[serde(default)] + pub caption: Option, +} + +impl SlideImage { + /// Identify and measure `bytes`, producing a consistent [`SlideImage`]. + /// + /// # Errors + /// + /// Returns [`Error::InvalidInput`] when `bytes` is empty, exceeds + /// [`MAX_IMAGE_BYTES`], is not PNG or JPEG, or carries a header this crate + /// cannot measure. + pub fn from_bytes(bytes: Vec, caption: Option) -> Result { + if bytes.is_empty() { + return Err(Error::invalid_input("bytes", "must not be empty")); + } + if bytes.len() > MAX_IMAGE_BYTES { + return Err(Error::invalid_input( + "bytes", + format!("must be ≤ {MAX_IMAGE_BYTES} bytes"), + )); + } + let format = ImageFormat::sniff(&bytes) + .ok_or_else(|| Error::invalid_input("bytes", "must be a PNG or JPEG image"))?; + let (width_px, height_px) = format.dimensions(&bytes).ok_or_else(|| { + Error::invalid_input( + "bytes", + format!("{format} header is truncated or malformed"), + ) + })?; + Ok(Self { + bytes, + format, + width_px, + height_px, + caption, + }) + } +} + +/// One content slide of the deck, rendered in spec order. +/// +/// At least one of `title`, `body`, or `bullets` must carry renderable text. +/// Images alone are not enough — a slide holding only an image and no label +/// reads as a rendering bug rather than a design choice, and synthesis drops +/// blank text anyway. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SlideSpec { + /// Slide title. May be blank for a visually minimal slide, as long as the + /// body or bullets carry text. + #[serde(default)] + pub title: String, + /// Body text, rendered above the bullets. Plain text only. + #[serde(default)] + pub body: Option, + /// Bullets, rendered after the body text. + #[serde(default)] + pub bullets: Vec, + /// Speaker notes attached to the slide. + #[serde(default)] + pub speaker_notes: Option, + /// Images, stacked in a single column beneath the text. + #[serde(default)] + pub images: Vec, +} + +impl SlideSpec { + /// Returns `true` when the slide carries no renderable text at all — the + /// title, body, and every bullet are absent or blank. + /// + /// Synthesis trims and drops blank entries, so a slide holding only + /// `[" "]` would render without text despite carrying entries. + #[must_use] + pub fn is_textless(&self) -> bool { + let has_title = !self.title.trim().is_empty(); + let has_body = self.body.as_deref().is_some_and(|b| !b.trim().is_empty()); + let has_bullets = self.bullets.iter().any(|b| !b.trim().is_empty()); + !(has_title || has_body || has_bullets) + } +} + +/// A complete `.pptx` presentation spec. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PresentationSpec { + /// Deck title, rendered on a leading title slide. Required and non-blank. + pub title: String, + /// Optional author byline, rendered beneath the deck title. + #[serde(default)] + pub author: Option, + /// Optional theme hint. + /// + /// Accepted and validated but not yet acted on: synthesis uses the writer's + /// default template regardless. It is part of the contract so a host's tool + /// schema does not have to change when template selection lands. + #[serde(default)] + pub theme: Option, + /// Content slides, in display order. Must contain at least one entry. + #[serde(default)] + pub slides: Vec, +} + +impl PresentationSpec { + /// Total number of images across every slide. + #[must_use] + pub fn image_count(&self) -> usize { + self.slides + .iter() + .map(|slide| slide.images.len()) + .sum::() + } + + /// Check the spec against every documented size limit, and check that each + /// image's declared format and dimensions match its bytes. + /// + /// Callers do not have to invoke this: `pptx::generate` validates before it + /// synthesises anything. It is public so a host can reject a malformed spec + /// at its own boundary — an LLM tool call, say — and hand back the + /// structured [`Error::InvalidInput`] before paying for a blocking hop, a + /// process boundary, or a bus round trip. + /// + /// # Errors + /// + /// Returns [`Error::InvalidInput`] naming the first field that violates a + /// limit. Fields are checked in spec order, so the reported field is stable + /// for a given spec. + pub fn validate(&self) -> Result<()> { + if self.title.trim().is_empty() { + return Err(Error::invalid_input("title", "must not be empty")); + } + Self::check_text_len("title", &self.title)?; + if let Some(author) = self.author.as_deref() { + Self::check_text_len("author", author)?; + } + if let Some(theme) = self.theme.as_deref() { + Self::check_text_len("theme", theme)?; + } + if self.slides.is_empty() { + return Err(Error::invalid_input( + "slides", + "must contain at least one slide", + )); + } + if self.slides.len() > MAX_SLIDES { + return Err(Error::invalid_input( + "slides", + format!("must contain ≤ {MAX_SLIDES} slides"), + )); + } + // Checked across the whole deck rather than per slide: the per-slide cap + // bounds readability, this one bounds the embedded media payload however + // the images are distributed. + if self.image_count() > MAX_IMAGES_PER_DECK { + return Err(Error::invalid_input( + "slides[].images", + format!("deck must contain ≤ {MAX_IMAGES_PER_DECK} images total"), + )); + } + + for (i, slide) in self.slides.iter().enumerate() { + if slide.is_textless() { + return Err(Error::invalid_input( + format!("slides[{i}]"), + "must have at least one of title / body / bullets", + )); + } + Self::check_text_len(format!("slides[{i}].title"), &slide.title)?; + if let Some(body) = slide.body.as_deref() { + Self::check_text_len(format!("slides[{i}].body"), body)?; + } + if slide.bullets.len() > MAX_BULLETS_PER_SLIDE { + return Err(Error::invalid_input( + format!("slides[{i}].bullets"), + format!("must contain ≤ {MAX_BULLETS_PER_SLIDE} bullets"), + )); + } + for (b, bullet) in slide.bullets.iter().enumerate() { + Self::check_text_len(format!("slides[{i}].bullets[{b}]"), bullet)?; + } + if let Some(notes) = slide.speaker_notes.as_deref() { + Self::check_text_len(format!("slides[{i}].speaker_notes"), notes)?; + } + if slide.images.len() > MAX_IMAGES_PER_SLIDE { + return Err(Error::invalid_input( + format!("slides[{i}].images"), + format!("must contain ≤ {MAX_IMAGES_PER_SLIDE} images"), + )); + } + for (m, image) in slide.images.iter().enumerate() { + Self::check_image(&format!("slides[{i}].images[{m}]"), image)?; + } + } + Ok(()) + } + + /// Reject a text field longer than [`MAX_TEXT_CHARS`] scalar values. + fn check_text_len(field: impl Into, value: &str) -> Result<()> { + if value.chars().count() > MAX_TEXT_CHARS { + return Err(Error::invalid_input( + field, + format!("must be ≤ {MAX_TEXT_CHARS} chars"), + )); + } + Ok(()) + } + + /// Re-derive an image's format and dimensions from its bytes and reject any + /// disagreement with what the spec declares. + /// + /// [`SlideImage::from_bytes`] keeps the fields consistent by construction, + /// but a spec can also arrive as deserialized JSON, where the three fields + /// are independent. A declared format that does not match the bytes yields + /// a part the reader refuses to render, and declared dimensions that do not + /// match distort the image silently — both are worth a named rejection. + fn check_image(field: &str, image: &SlideImage) -> Result<()> { + if image.bytes.is_empty() { + return Err(Error::invalid_input( + format!("{field}.bytes"), + "must not be empty", + )); + } + if image.bytes.len() > MAX_IMAGE_BYTES { + return Err(Error::invalid_input( + format!("{field}.bytes"), + format!("must be ≤ {MAX_IMAGE_BYTES} bytes"), + )); + } + let sniffed = ImageFormat::sniff(&image.bytes).ok_or_else(|| { + Error::invalid_input(format!("{field}.bytes"), "must be a PNG or JPEG image") + })?; + if sniffed != image.format { + return Err(Error::invalid_input( + format!("{field}.format"), + format!("declared {} but the bytes are {sniffed}", image.format), + )); + } + let (width_px, height_px) = sniffed.dimensions(&image.bytes).ok_or_else(|| { + Error::invalid_input( + format!("{field}.bytes"), + format!("{sniffed} header is truncated or malformed"), + ) + })?; + if (width_px, height_px) != (image.width_px, image.height_px) { + return Err(Error::invalid_input( + format!("{field}.width_px"), + format!( + "declared {}x{} but the bytes are {width_px}x{height_px}", + image.width_px, image.height_px + ), + )); + } + if let Some(caption) = image.caption.as_deref() { + Self::check_text_len(format!("{field}.caption"), caption)?; + } + Ok(()) + } +} + +pub mod wire; + +#[cfg(test)] +mod test; diff --git a/src/openhuman/tools/impl/document/format/spec/presentation/test.rs b/src/openhuman/tools/impl/document/format/spec/presentation/test.rs new file mode 100644 index 0000000000..9dcfef7fd3 --- /dev/null +++ b/src/openhuman/tools/impl/document/format/spec/presentation/test.rs @@ -0,0 +1,367 @@ +//! Unit tests for the presentation wire contract. +//! +//! Format-independent, like the spec itself: these must pass in a build with +//! every format feature off. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{ + PresentationSpec, SlideImage, SlideSpec, MAX_BULLETS_PER_SLIDE, MAX_IMAGES_PER_DECK, + MAX_IMAGES_PER_SLIDE, MAX_IMAGE_BYTES, MAX_SLIDES, MAX_TEXT_CHARS, +}; +use crate::openhuman::tools::implementations::document::format::spec::image::test::{jpeg, png}; +use crate::openhuman::tools::implementations::document::format::spec::image::ImageFormat; +use crate::openhuman::tools::implementations::document::format::Error; + +/// One valid slide carrying a title, a body, and a bullet. +fn slide() -> SlideSpec { + SlideSpec { + title: "Overview".to_string(), + body: Some("The situation so far.".to_string()), + bullets: vec!["A bullet".to_string()], + speaker_notes: Some("Keep it short.".to_string()), + images: vec![], + } +} + +/// A minimal valid spec; each test mutates one field to drive a single branch. +fn spec() -> PresentationSpec { + PresentationSpec { + title: "Quarterly Review".to_string(), + author: Some("Alice".to_string()), + theme: Some("plain".to_string()), + slides: vec![slide()], + } +} + +/// A valid image built from real header bytes. +fn image() -> SlideImage { + SlideImage::from_bytes(png(320, 200), Some("A chart".to_string())).expect("valid png") +} + +/// Assert `spec` is rejected with an `InvalidInput` naming `field`. +fn assert_rejects(spec: &PresentationSpec, field: &str) { + match spec.validate() { + Err(Error::InvalidInput { field: f, .. }) => { + assert_eq!(f, field, "unexpected rejected field"); + } + other => panic!("expected InvalidInput({field}), got {other:?}"), + } +} + +#[test] +fn accepts_a_well_formed_spec() { + assert!(spec().validate().is_ok()); +} + +#[test] +fn accepts_a_spec_with_images() { + let mut s = spec(); + s.slides[0].images = vec![image()]; + assert!(s.validate().is_ok()); +} + +#[test] +fn rejects_a_blank_deck_title() { + let mut s = spec(); + s.title = " ".to_string(); + assert_rejects(&s, "title"); +} + +#[test] +fn rejects_over_long_deck_level_text() { + for (field, mutate) in [("title", 0), ("author", 1), ("theme", 2)] { + let mut s = spec(); + let long = "x".repeat(MAX_TEXT_CHARS + 1); + match mutate { + 0 => s.title = long, + 1 => s.author = Some(long), + _ => s.theme = Some(long), + } + assert_rejects(&s, field); + } +} + +#[test] +fn rejects_a_spec_with_no_slides() { + let mut s = spec(); + s.slides.clear(); + assert_rejects(&s, "slides"); +} + +#[test] +fn rejects_too_many_slides() { + let mut s = spec(); + s.slides = vec![slide(); MAX_SLIDES + 1]; + assert_rejects(&s, "slides"); +} + +#[test] +fn rejects_a_textless_slide() { + // Every text entry is present but whitespace-only, so synthesis would drop + // all of them and render an unlabelled slide. + let mut s = spec(); + s.slides = vec![SlideSpec { + title: " ".to_string(), + body: Some("\t".to_string()), + bullets: vec![String::new()], + speaker_notes: None, + images: vec![], + }]; + assert_rejects(&s, "slides[0]"); +} + +#[test] +fn rejects_a_slide_carrying_only_an_image() { + // Images do not satisfy the "must have text" rule: an unlabelled slide + // reads as a rendering bug rather than a design choice. + let mut s = spec(); + s.slides = vec![SlideSpec { + title: String::new(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![image()], + }]; + assert_rejects(&s, "slides[0]"); +} + +#[test] +fn rejects_over_long_slide_text_naming_its_index() { + let long = || "x".repeat(MAX_TEXT_CHARS + 1); + + let mut s = spec(); + s.slides.push(SlideSpec { + title: long(), + ..slide() + }); + assert_rejects(&s, "slides[1].title"); + + let mut s = spec(); + s.slides[0].body = Some(long()); + assert_rejects(&s, "slides[0].body"); + + let mut s = spec(); + s.slides[0].bullets = vec!["ok".to_string(), long()]; + assert_rejects(&s, "slides[0].bullets[1]"); + + let mut s = spec(); + s.slides[0].speaker_notes = Some(long()); + assert_rejects(&s, "slides[0].speaker_notes"); +} + +#[test] +fn rejects_too_many_bullets() { + let mut s = spec(); + s.slides[0].bullets = vec!["b".to_string(); MAX_BULLETS_PER_SLIDE + 1]; + assert_rejects(&s, "slides[0].bullets"); +} + +#[test] +fn rejects_too_many_images_on_one_slide() { + let mut s = spec(); + s.slides[0].images = vec![image(); MAX_IMAGES_PER_SLIDE + 1]; + assert_rejects(&s, "slides[0].images"); +} + +#[test] +fn rejects_too_many_images_across_the_deck() { + // Each slide is within the per-slide cap; only the deck total is not. The + // per-slide cap bounds readability, the deck cap bounds the media payload. + let per_slide = MAX_IMAGES_PER_SLIDE; + let slides_needed = MAX_IMAGES_PER_DECK / per_slide + 1; + let mut s = spec(); + s.slides = vec![ + SlideSpec { + images: vec![image(); per_slide], + ..slide() + }; + slides_needed + ]; + assert!(s.image_count() > MAX_IMAGES_PER_DECK); + assert_rejects(&s, "slides[].images"); +} + +#[test] +fn image_count_sums_across_slides() { + let mut s = spec(); + s.slides = vec![ + SlideSpec { + images: vec![image(), image()], + ..slide() + }, + SlideSpec { + images: vec![image()], + ..slide() + }, + ]; + assert_eq!(s.image_count(), 3); +} + +#[test] +fn rejects_an_over_long_image_caption() { + let mut s = spec(); + let mut img = image(); + img.caption = Some("c".repeat(MAX_TEXT_CHARS + 1)); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].caption"); +} + +#[test] +fn from_bytes_derives_format_and_dimensions() { + let img = SlideImage::from_bytes(png(1920, 1080), None).expect("valid png"); + assert_eq!(img.format, ImageFormat::Png); + assert_eq!((img.width_px, img.height_px), (1920, 1080)); + assert_eq!(img.caption, None); + + let img = SlideImage::from_bytes(jpeg(640, 480), Some("j".to_string())).expect("valid jpeg"); + assert_eq!(img.format, ImageFormat::Jpeg); + assert_eq!((img.width_px, img.height_px), (640, 480)); +} + +#[test] +fn from_bytes_rejects_bad_input() { + assert!(matches!( + SlideImage::from_bytes(vec![], None), + Err(Error::InvalidInput { .. }) + )); + assert!(matches!( + SlideImage::from_bytes(b"not an image".to_vec(), None), + Err(Error::InvalidInput { .. }) + )); + // PNG signature with a truncated IHDR: the right format, unmeasurable. + assert!(matches!( + SlideImage::from_bytes(vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], None), + Err(Error::InvalidInput { .. }) + )); +} + +#[test] +fn from_bytes_rejects_an_oversize_image() { + // A real PNG header followed by enough filler to cross the cap, so the + // rejection is the size check rather than the sniff. + let mut bytes = png(8, 8); + bytes.resize(MAX_IMAGE_BYTES + 1, 0); + assert!(matches!( + SlideImage::from_bytes(bytes, None), + Err(Error::InvalidInput { .. }) + )); +} + +#[test] +fn validate_rejects_an_image_whose_declared_format_contradicts_its_bytes() { + // `from_bytes` cannot produce this, but deserialized JSON can: the three + // fields are independent on the wire. A wrong format yields a part the + // reader refuses to render, so it is worth a named rejection. + let mut s = spec(); + let mut img = image(); + img.format = ImageFormat::Jpeg; + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].format"); +} + +#[test] +fn validate_rejects_an_image_whose_declared_dimensions_contradict_its_bytes() { + // Declared dimensions that disagree with the bytes distort the image + // silently, which is worse than failing. + let mut s = spec(); + let mut img = image(); + img.width_px += 1; + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].width_px"); +} + +#[test] +fn validate_rejects_empty_oversize_and_unrecognised_image_bytes() { + let mut s = spec(); + let mut img = image(); + img.bytes.clear(); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].bytes"); + + let mut s = spec(); + let mut img = image(); + img.bytes = b"not an image".to_vec(); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].bytes"); + + let mut s = spec(); + let mut img = image(); + img.bytes.resize(MAX_IMAGE_BYTES + 1, 0); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].bytes"); +} + +#[test] +fn validate_rejects_an_image_with_an_unmeasurable_header() { + // Sniffs as PNG, but the IHDR is gone — measurement fails after the format + // check has already passed, which is a distinct branch. + let mut s = spec(); + let mut img = image(); + img.bytes.truncate(8); + s.slides[0].images = vec![img]; + assert_rejects(&s, "slides[0].images[0].bytes"); +} + +#[test] +fn is_textless_reflects_text_presence() { + assert!(!slide().is_textless()); + assert!(SlideSpec { + title: String::new(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![], + } + .is_textless()); + // A title alone is enough. + assert!(!SlideSpec { + title: "Only a title".to_string(), + body: None, + bullets: vec![], + speaker_notes: None, + images: vec![], + } + .is_textless()); + // So is a body alone, or a bullet alone. + assert!(!SlideSpec { + title: String::new(), + body: Some("Body".to_string()), + bullets: vec![], + speaker_notes: None, + images: vec![], + } + .is_textless()); + assert!(!SlideSpec { + title: String::new(), + body: None, + bullets: vec!["Bullet".to_string()], + speaker_notes: None, + images: vec![], + } + .is_textless()); +} + +#[test] +fn spec_round_trips_through_json() { + let mut s = spec(); + s.slides[0].images = vec![image()]; + let json = serde_json::to_string(&s).expect("serialises"); + let back: PresentationSpec = serde_json::from_str(&json).expect("deserialises"); + assert_eq!(back, s); + assert!(back.validate().is_ok()); +} + +#[test] +fn spec_rejects_unknown_json_fields() { + let json = r#"{"title":"T","slides":[],"tilte":"typo"}"#; + assert!(serde_json::from_str::(json).is_err()); +} + +#[test] +fn spec_defaults_optional_fields() { + let s: PresentationSpec = serde_json::from_str(r#"{"title":"T"}"#).expect("deserialises"); + assert_eq!(s.author, None); + assert_eq!(s.theme, None); + assert!(s.slides.is_empty()); +} diff --git a/src/openhuman/tools/impl/document/format/spec/presentation/wire.rs b/src/openhuman/tools/impl/document/format/spec/presentation/wire.rs new file mode 100644 index 0000000000..ef9cb8504c --- /dev/null +++ b/src/openhuman/tools/impl/document/format/spec/presentation/wire.rs @@ -0,0 +1,73 @@ +//! The presentation spec as it crosses a bus, where bytes cannot travel inline. +//! +//! A `TinyBus` frame is a 16 MiB JSON document and a deck may legally carry +//! 40 MiB of images, so image bytes ride a stream beside the call rather than +//! inside it. A call has one stream and a deck has many images, so the images +//! are concatenated in slide order and each one declares its `byte_len`; the +//! module splits them apart and resolves each into a real +//! [`super::SlideImage`] — bytes, format and dimensions. +//! +//! The lengths live in the spec rather than in the stream because they are what +//! makes a truncated or over-long transfer a named rejection instead of a deck +//! with a picture assembled from two different images. +//! +//! Only the presentation spec needs this treatment. A document spec is text and +//! its aggregate cap keeps it inside a frame, so a document crosses unchanged. +//! +//! Defined here rather than in the module that serves it so a host driving that +//! module over a bus shares one definition of the shape instead of re-declaring +//! it. Like the rest of [`crate::openhuman::tools::implementations::document::format::spec`] it is serde and nothing else. + +use serde::{Deserialize, Serialize}; + +/// A slide image, as it appears on the bus: one byte range of the concatenated +/// image stream that travels beside the call. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WireSlideImage { + /// Length of this image's bytes within the concatenated image stream. + pub byte_len: u64, + /// Optional caption, rendered as a bullet beneath the image. + #[serde(default)] + pub caption: Option, +} + +/// One content slide, as it appears on the bus. +/// +/// Identical to [`super::SlideSpec`] apart from `images`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WireSlideSpec { + /// Slide title. + #[serde(default)] + pub title: String, + /// Body text, rendered above the bullets. + #[serde(default)] + pub body: Option, + /// Bullets, rendered after the body text. + #[serde(default)] + pub bullets: Vec, + /// Speaker notes attached to the slide. + #[serde(default)] + pub speaker_notes: Option, + /// Images, each naming a staged blob. + #[serde(default)] + pub images: Vec, +} + +/// A deck, as it appears on the bus. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WirePresentationSpec { + /// Deck title, rendered on a leading title slide. + pub title: String, + /// Optional author byline. + #[serde(default)] + pub author: Option, + /// Optional theme hint. + #[serde(default)] + pub theme: Option, + /// Content slides, in display order. + #[serde(default)] + pub slides: Vec, +} diff --git a/src/openhuman/tools/impl/document/mod.rs b/src/openhuman/tools/impl/document/mod.rs index 2ccae1e399..a221fc01ef 100644 --- a/src/openhuman/tools/impl/document/mod.rs +++ b/src/openhuman/tools/impl/document/mod.rs @@ -39,6 +39,7 @@ use crate::openhuman::security::SecurityPolicy; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; mod engine; +pub(crate) mod format; mod types; #[cfg(test)] diff --git a/src/openhuman/tools/impl/document/types.rs b/src/openhuman/tools/impl/document/types.rs index fd7eefb9c4..6614f215e8 100644 --- a/src/openhuman/tools/impl/document/types.rs +++ b/src/openhuman/tools/impl/document/types.rs @@ -1,20 +1,20 @@ //! Typed input / output / error contracts for the `generate_document` tool. //! //! The *document* half of these contracts — the section spec, its size -//! limits, and the validation rules — lives in the vendored -//! [`tinydocs`](https://github.com/tinyhumansai/tinydocs) crate and is -//! re-exported here. Nothing about "a title, some sections, and a bullet +//! limits, and the validation rules — lives in the host-local +//! [`format`](super::format) +//! module and is re-exported here. Nothing about "a title, some sections, and a bullet //! list" is OpenHuman-specific, so the definitions live where any host can //! reach them and this module keeps only what genuinely is ours: //! //! - [`GenerateDocumentOutput`] — artifact ids and workspace paths, concepts -//! `tinydocs` has no notion of. +//! the bus contract has no notion of. //! - [`DocumentError`] — the agent-facing error shape, which carries a -//! [`DocumentError::GenerationTimeout`] variant `tinydocs` cannot produce: +//! [`DocumentError::GenerationTimeout`] variant the document module cannot produce: //! the deadline is OpenHuman's policy, applied by [`engine`](super::engine) //! around a synchronous crate call. //! -//! The re-exported [`GenerateDocumentInput`] is `tinydocs`' `DocumentSpec` +//! The re-exported [`GenerateDocumentInput`] is the format module's `DocumentSpec` //! under its historical OpenHuman name. Field names are unchanged, so the JSON //! tool schema the agent sees is byte-identical to before the extraction. @@ -26,7 +26,7 @@ use crate::openhuman::modules::documents::DocumentCallError; // the writer, so the gated `docx` module is not compiled here at all. The types // are the same ones the module validates against, which is the whole reason // `spec` is separable. -pub use tinydocs::spec::document::{ +pub use crate::openhuman::tools::implementations::document::format::spec::document::{ DocumentSpec as GenerateDocumentInput, MAX_BULLETS_PER_SECTION, MAX_PARAGRAPHS_PER_SECTION, MAX_PARAGRAPH_CHARS, MAX_SECTIONS, MAX_TEXT_CHARS, }; @@ -37,7 +37,7 @@ pub use tinydocs::spec::document::{ // is private, so the re-export reads as unused to the compiler — hence the // explicit allow rather than dropping a name callers legitimately need. #[allow(unused_imports)] -pub use tinydocs::spec::DocumentSection; +pub use crate::openhuman::tools::implementations::document::format::spec::DocumentSection; /// Tool output returned via [`crate::openhuman::tools::traits::ToolResult`] /// as the JSON `data` field. @@ -81,11 +81,11 @@ impl DocumentError { /// the variant never carries an unbounded payload back to the agent. /// Same cap/suffix as the presentation tool's `truncate_stderr`. pub(super) fn truncate_stderr(raw: &str) -> String { - tinydocs::Error::truncate_detail(raw) + crate::openhuman::tools::implementations::document::format::Error::truncate_detail(raw) } } -impl From for DocumentError { +impl From for DocumentError { /// Map a spec-validation failure onto the agent-facing shape. /// /// This is the *local* path: `validate_input` below checks the spec against @@ -98,15 +98,15 @@ impl From for DocumentError { /// `GenerationTimeout` is deliberately absent: validation has no deadline, /// so only [`engine`](super::engine) produces that variant. /// - /// `tinydocs::Error` is `#[non_exhaustive]`, so the catch-all arm is + /// `crate::openhuman::tools::implementations::document::format::Error` is `#[non_exhaustive]`, so the catch-all arm is /// required by the compiler rather than chosen. It degrades a variant added /// by a future release to `GenerationFailed` carrying that variant's own /// `Display` text, and logs, so a crate bump that introduces a case worth /// handling structurally shows up rather than being swallowed. - fn from(err: tinydocs::Error) -> Self { + fn from(err: crate::openhuman::tools::implementations::document::format::Error) -> Self { match err { - tinydocs::Error::InvalidInput { field, reason } => Self::InvalidInput { field, reason }, - tinydocs::Error::GenerationFailed { detail } => Self::GenerationFailed { + crate::openhuman::tools::implementations::document::format::Error::InvalidInput { field, reason } => Self::InvalidInput { field, reason }, + crate::openhuman::tools::implementations::document::format::Error::GenerationFailed { detail } => Self::GenerationFailed { stderr_truncated: detail, }, other => { @@ -166,7 +166,7 @@ impl From for DocumentError { /// generic engine error. /// /// Delegates to `tinydocs`, which validates again inside -/// [`tinydocs::docx::generate`]. The double check is intentional: validating +/// [`crate::openhuman::tools::implementations::document::format::docx::generate`]. The double check is intentional: validating /// here lets the tool reject a bad call before allocating an artifact record, /// and the crate-side check keeps `generate` safe for any other caller. pub(super) fn validate_input(input: &GenerateDocumentInput) -> Result<(), DocumentError> { @@ -300,10 +300,12 @@ mod tests { fn tinydocs_invalid_input_keeps_its_field_and_reason() { // The structured pair is what the agent self-corrects on, so the // crate-boundary mapping must not flatten it into a message string. - let mapped = DocumentError::from(tinydocs::Error::InvalidInput { - field: "sections[3].bullets[1]".to_string(), - reason: "must be ≤ 20000 chars".to_string(), - }); + let mapped = DocumentError::from( + crate::openhuman::tools::implementations::document::format::Error::InvalidInput { + field: "sections[3].bullets[1]".to_string(), + reason: "must be ≤ 20000 chars".to_string(), + }, + ); match mapped { DocumentError::InvalidInput { field, reason } => { assert_eq!(field, "sections[3].bullets[1]"); @@ -317,10 +319,15 @@ mod tests { fn tinydocs_generation_failure_maps_without_re_truncating() { // `tinydocs` already truncated this detail; re-truncating would eat // the suffix and misreport how much was dropped. - let detail = tinydocs::Error::truncate_detail(&"x".repeat(10_000)); - let mapped = DocumentError::from(tinydocs::Error::GenerationFailed { - detail: detail.clone(), - }); + let detail = + crate::openhuman::tools::implementations::document::format::Error::truncate_detail( + &"x".repeat(10_000), + ); + let mapped = DocumentError::from( + crate::openhuman::tools::implementations::document::format::Error::GenerationFailed { + detail: detail.clone(), + }, + ); match mapped { DocumentError::GenerationFailed { stderr_truncated } => { assert_eq!(stderr_truncated, detail); @@ -332,7 +339,10 @@ mod tests { #[test] fn truncate_stderr_bounds_the_payload() { let out = DocumentError::truncate_stderr(&"x".repeat(10_000)); - assert_eq!(out.chars().count(), tinydocs::Error::MAX_DETAIL_CHARS); + assert_eq!( + out.chars().count(), + crate::openhuman::tools::implementations::document::format::Error::MAX_DETAIL_CHARS + ); assert!(out.ends_with("[…truncated]")); } diff --git a/src/openhuman/tools/impl/presentation/engine.rs b/src/openhuman/tools/impl/presentation/engine.rs index b33e8663cb..a1fdd376e1 100644 --- a/src/openhuman/tools/impl/presentation/engine.rs +++ b/src/openhuman/tools/impl/presentation/engine.rs @@ -1,7 +1,7 @@ //! Async wrapper around the `tinydocs` module's `.pptx` writer. //! //! The synthesis itself — the slide mapping, the single-column image layout, the -//! EMU geometry — lives in `tinydocs::pptx` and runs inside the loaded module. +//! EMU geometry — lives in `crate::openhuman::tools::implementations::document::format::pptx` and runs inside the loaded module. //! What is left here is the policy only a host can supply: //! //! 1. a deadline, because the module holds no opinion about how long a caller @@ -24,7 +24,9 @@ use std::time::Duration; -use tinydocs::spec::{WirePresentationSpec, WireSlideImage, WireSlideSpec}; +use crate::openhuman::tools::implementations::document::format::spec::{ + WirePresentationSpec, WireSlideImage, WireSlideSpec, +}; use tokio::time::timeout; use super::types::{GeneratePresentationInput, PresentationError, ResolvedSlideImage}; @@ -172,7 +174,7 @@ mod tests { //! What is left to test on this side of the bus. //! //! The deck shape, the image layout and the OOXML container are tested in - //! `tinydocs::pptx`, where the code now lives — reproducing them here would + //! `crate::openhuman::tools::implementations::document::format::pptx`, where the code now lives — reproducing them here would //! assert the same behaviour twice and drift the moment one copy changed. //! //! What only exists here is [`build_request`]: the deck and the concatenated @@ -204,7 +206,8 @@ mod tests { fn resolved(bytes: &[u8], caption: Option<&str>) -> ResolvedSlideImage { ResolvedSlideImage { bytes: bytes.to_vec(), - format: tinydocs::spec::ImageFormat::Png, + format: + crate::openhuman::tools::implementations::document::format::spec::ImageFormat::Png, width_px: 4, height_px: 4, caption: caption.map(str::to_string), diff --git a/src/openhuman/tools/impl/presentation/mod.rs b/src/openhuman/tools/impl/presentation/mod.rs index 1635821080..b833eb1810 100644 --- a/src/openhuman/tools/impl/presentation/mod.rs +++ b/src/openhuman/tools/impl/presentation/mod.rs @@ -27,10 +27,10 @@ //! #3026 Files panel, and the orchestrator grounding rule in #3029 //! continue to work without change. +use crate::openhuman::tools::implementations::document::format::spec::ImageFormat; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -use tinydocs::spec::ImageFormat; use async_trait::async_trait; use serde_json::{json, Value}; @@ -455,7 +455,7 @@ impl PresentationTool { )); } - // Identification and measurement live in `tinydocs::spec::image`, which + // Identification and measurement live in `crate::openhuman::tools::implementations::document::format::spec::image`, which // is ungated: a host resolving image bytes has to do this to build a // spec, and it must not need the writer to do it. One implementation // also means the host and the module cannot disagree about what is diff --git a/src/openhuman/tools/impl/presentation/types.rs b/src/openhuman/tools/impl/presentation/types.rs index 051cbb0c1d..54a3119f78 100644 --- a/src/openhuman/tools/impl/presentation/types.rs +++ b/src/openhuman/tools/impl/presentation/types.rs @@ -1,7 +1,7 @@ //! Typed input / output / error contracts for the `generate_presentation` tool. +use crate::openhuman::tools::implementations::document::format::spec::ImageFormat; use serde::{Deserialize, Serialize}; -use tinydocs::spec::ImageFormat; use crate::openhuman::modules::documents::DocumentCallError; diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index 5b83a3bd92..3a17fe29dc 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -1568,8 +1568,8 @@ fn tool_group(name: &str) -> crate::core::all::DomainGroup { /// about what the *model is told exists*, and the later re-point onto /// `MemoryGuard` must not change the advertised surface. Assigning them `None` /// to dodge the mismatch would bake the wrong contract in. -fn tool_capability(name: &str) -> Option { - use tinycortex_api::capabilities::Capability; +fn tool_capability(name: &str) -> Option { + use crate::openhuman::memory::api::capabilities::Capability; // Not driver-backed. Each entry is an argued exception, not a fallthrough. if name == "update_memory_md" // writes the workspace `MEMORY.md` file directly diff --git a/src/openhuman/tools/ops_tests.rs b/src/openhuman/tools/ops_tests.rs index 41338db2cd..79d153b890 100644 --- a/src/openhuman/tools/ops_tests.rs +++ b/src/openhuman/tools/ops_tests.rs @@ -2808,8 +2808,11 @@ const TOOL_LESS: &[crate::core::all::DomainGroup] = { // ---- tool_capability() drift guard (M5.3) ---------------------------------- /// Driver-backed memory tools and the capability each requires. -const MEMORY_TOOL_CAPABILITIES: &[(&str, tinycortex_api::capabilities::Capability)] = { - use tinycortex_api::capabilities::Capability as C; +const MEMORY_TOOL_CAPABILITIES: &[( + &str, + crate::openhuman::memory::api::capabilities::Capability, +)] = { + use crate::openhuman::memory::api::capabilities::Capability as C; &[ ("memory_store", C::Core), ("memory_forget", C::Core), @@ -2896,7 +2899,7 @@ fn every_memory_tool_has_an_explicit_capability_or_is_core() { /// (the never-filtered bucket). Synthetic names matching only the prefix. #[test] fn no_prefix_family_memory_tool_silently_defaults_to_uncapped() { - use tinycortex_api::capabilities::Capability; + use crate::openhuman::memory::api::capabilities::Capability; for (name, want) in [ ("goals_new_thing", Capability::Goals), ("memory_tree_new_thing", Capability::Tree), @@ -3002,10 +3005,11 @@ fn memory_tools_all_present_with_no_ambient_context() { } } -/// Under the default (`driver = "tinycortex"`) binding the embedded driver +/// Under the default binding the TinyMemory module /// advertises all thirteen families, so the list is byte-identical to today. #[tokio::test] -async fn memory_tools_all_present_under_the_embedded_driver() { +#[cfg(feature = "modules")] +async fn memory_tools_all_present_under_the_module_driver() { use crate::core::runtime::context::CoreContext; use crate::core::runtime::DomainSet; @@ -3022,13 +3026,13 @@ async fn memory_tools_all_present_under_the_embedded_driver() { { assert!( names.iter().any(|n| n == name), - "`{name}` must survive the embedded driver; got: {names:?}" + "`{name}` must survive the module driver; got: {names:?}" ); } if cfg!(feature = "memory-git") { assert!( names.iter().any(|n| n == "memory_diff"), - "`memory_diff` must survive the embedded driver when `memory-git` is on; got: {names:?}" + "`memory_diff` must survive the module driver when `memory-git` is on; got: {names:?}" ); } } diff --git a/src/openhuman/web3/wallet/abi.rs b/src/openhuman/web3/wallet/abi.rs index 9cac177438..0597c0ae41 100644 --- a/src/openhuman/web3/wallet/abi.rs +++ b/src/openhuman/web3/wallet/abi.rs @@ -5,7 +5,7 @@ //! grammar, a bignum, and their tails — to produce a four-byte selector //! followed by two 32-byte words. //! -//! `tinywallet::abi` owns that encoding now, over `sha3` alone, and +//! `crate::openhuman::web3::wallet::primitives::abi` owns that encoding now, over `sha3` alone, and //! deliberately sits outside its `tx` gate: calldata is an *input* to building //! a transaction, so a host that builds elsewhere still needs it locally rather //! than paying a bus round trip for keccak over 68 bytes. @@ -25,19 +25,21 @@ /// /// A human-readable message if the recipient is not a valid EVM address or the /// amount is not a non-negative integer that fits in 256 bits. +#[allow(unreachable_patterns)] pub fn encode_erc20_transfer(to_address: &str, amount_raw: &str) -> Result { - tinywallet::abi::encode_erc20_transfer(to_address, amount_raw).map_err(|error| match error { - tinywallet::abi::Error::InvalidRecipient { .. } => { - format!("invalid EVM recipient address '{to_address}': {error}") - } - // Preserves the wording the previous implementation used, because the - // agent tool's schema documents it and a model reads it to correct - // itself. - tinywallet::abi::Error::InvalidAmount { .. } => { - format!("amount '{amount_raw}' is not a valid non-negative integer") - } - _ => error.to_string(), - }) + crate::openhuman::web3::wallet::primitives::abi::encode_erc20_transfer(to_address, amount_raw) + .map_err(|error| match error { + crate::openhuman::web3::wallet::primitives::abi::Error::InvalidRecipient { .. } => { + format!("invalid EVM recipient address '{to_address}': {error}") + } + // Preserves the wording the previous implementation used, because the + // agent tool's schema documents it and a model reads it to correct + // itself. + crate::openhuman::web3::wallet::primitives::abi::Error::InvalidAmount { .. } => { + format!("amount '{amount_raw}' is not a valid non-negative integer") + } + _ => error.to_string(), + }) } #[cfg(test)] diff --git a/src/openhuman/web3/wallet/chains/btc.rs b/src/openhuman/web3/wallet/chains/btc.rs index c025836fd7..c35ad94b55 100644 --- a/src/openhuman/web3/wallet/chains/btc.rs +++ b/src/openhuman/web3/wallet/chains/btc.rs @@ -61,7 +61,8 @@ pub fn estimated_btc_fee_sats() -> u64 { /// specific, so the rules live where any host can reach them; what stays here /// is the `Result<_, String>` shape the rest of this domain speaks. pub fn validate_btc_address(addr: &str) -> Result { - let result = tinywallet::address::btc::validate(addr).map_err(|e| e.to_string()); + let result = crate::openhuman::web3::wallet::primitives::address::btc::validate(addr) + .map_err(|e| e.to_string()); debug!( "{LOG_PREFIX} validate_address role=recipient result={}", if result.is_ok() { @@ -81,7 +82,8 @@ pub fn validate_btc_address(addr: &str) -> Result { /// the recipient rule for a sender accepts an address that only fails later, /// at signing time. pub fn validate_btc_sender_address(addr: &str) -> Result { - let result = tinywallet::address::btc::validate_sender(addr).map_err(|e| e.to_string()); + let result = crate::openhuman::web3::wallet::primitives::address::btc::validate_sender(addr) + .map_err(|e| e.to_string()); debug!( "{LOG_PREFIX} validate_address role=sender result={}", if result.is_ok() { @@ -130,8 +132,12 @@ fn derive_btc_private_key( mnemonic: &str, derivation_path: &str, ) -> Result<(Vec, Vec), String> { - let derived = tinywallet::key::derive(tinywallet::Chain::Btc, mnemonic, derivation_path) - .map_err(|e| e.to_string())?; + let derived = crate::openhuman::web3::wallet::primitives::key::derive( + crate::openhuman::web3::wallet::primitives::Chain::Btc, + mnemonic, + derivation_path, + ) + .map_err(|e| e.to_string())?; let secret = derived.secret_bytes().to_vec(); // Compressed, because a P2WPKH witness program is defined over the // compressed encoding — the uncompressed form yields a valid-looking @@ -209,18 +215,20 @@ pub async fn execute_btc_quote(mut quote: PreparedTransaction) -> Result Result ( - tinywallet::address::evm::validate("e.to_address).map_err(|e| { - format!("invalid EVM recipient address '{}': {e}", quote.to_address) - })?, + crate::openhuman::web3::wallet::primitives::address::evm::validate("e.to_address) + .map_err(|e| format!("invalid EVM recipient address '{}': {e}", quote.to_address))?, quote.amount_raw.clone(), None, ), @@ -158,7 +157,7 @@ pub async fn execute_evm_quote(mut quote: PreparedTransaction) -> Result Result Result { - let result = tinywallet::address::evm::validate(addr).map_err(|e| e.to_string()); + let result = crate::openhuman::web3::wallet::primitives::address::evm::validate(addr) + .map_err(|e| e.to_string()); debug!( "{LOG_PREFIX} validate_address result={}", if result.is_ok() { diff --git a/src/openhuman/web3/wallet/chains/solana.rs b/src/openhuman/web3/wallet/chains/solana.rs index aa81813626..b005131b3f 100644 --- a/src/openhuman/web3/wallet/chains/solana.rs +++ b/src/openhuman/web3/wallet/chains/solana.rs @@ -66,7 +66,8 @@ struct BlockhashValue { /// format; this wrapper keeps the `Result<_, String>` shape the rest of the /// domain speaks. pub fn validate_solana_address(addr: &str) -> Result { - let result = tinywallet::address::solana::validate(addr).map_err(|e| e.to_string()); + let result = crate::openhuman::web3::wallet::primitives::address::solana::validate(addr) + .map_err(|e| e.to_string()); debug!( "{LOG_PREFIX} validate_address result={}", if result.is_ok() { @@ -103,8 +104,12 @@ pub async fn native_balance(address: &str) -> Result { /// because such a path is derivable-looking but underivable on ed25519 — and /// silently hardening it would return a different account than the path names. fn derive_solana_keypair(mnemonic: &str, derivation_path: &str) -> Result { - let derived = tinywallet::key::derive(tinywallet::Chain::Solana, mnemonic, derivation_path) - .map_err(|e| e.to_string())?; + let derived = crate::openhuman::web3::wallet::primitives::key::derive( + crate::openhuman::web3::wallet::primitives::Chain::Solana, + mnemonic, + derivation_path, + ) + .map_err(|e| e.to_string())?; let bytes: [u8; SECRET_KEY_LENGTH] = derived .secret_bytes() .try_into() diff --git a/src/openhuman/web3/wallet/chains/tron.rs b/src/openhuman/web3/wallet/chains/tron.rs index f5f5e3b51f..3e67e4bae7 100644 --- a/src/openhuman/web3/wallet/chains/tron.rs +++ b/src/openhuman/web3/wallet/chains/tron.rs @@ -32,7 +32,8 @@ const TRC20_FEE_LIMIT_SUN: u64 = 15_000_000; /// format; this wrapper keeps the `Result<_, String>` shape the rest of the /// domain speaks. pub fn validate_tron_address(addr: &str) -> Result { - let result = tinywallet::address::tron::validate(addr).map_err(|e| e.to_string()); + let result = crate::openhuman::web3::wallet::primitives::address::tron::validate(addr) + .map_err(|e| e.to_string()); debug!( "{LOG_PREFIX} validate_address result={}", if result.is_ok() { @@ -53,7 +54,8 @@ pub fn validate_tron_address(addr: &str) -> Result { /// the wrong length used to produce a short hex string and fail further /// downstream at the API call. pub fn tron_address_to_hex(addr: &str) -> Result { - let result = tinywallet::address::tron::to_hex(addr).map_err(|e| e.to_string()); + let result = crate::openhuman::web3::wallet::primitives::address::tron::to_hex(addr) + .map_err(|e| e.to_string()); debug!( "{LOG_PREFIX} address_to_hex result={}", if result.is_ok() { @@ -114,7 +116,7 @@ fn tron_transaction_spec( raw_tx: &CreateTransactionResponse, expected_to: String, transfer: &TronTransferVerification, -) -> Result { +) -> Result { let recomputed_txid = recompute_tron_txid(&raw_tx.raw_data_hex)?; if !recomputed_txid.eq_ignore_ascii_case(raw_tx.tx_id.trim()) { return Err("Tron node txID does not match sha256(raw_data)".to_string()); @@ -174,11 +176,13 @@ fn tron_transaction_spec( } } - Ok(tinywallet::wire::TransactionSpec::Tron { - raw_data_hex: raw_tx.raw_data_hex.clone(), - expected_to, - expected_txid: recomputed_txid, - }) + Ok( + crate::openhuman::web3::wallet::primitives::wire::TransactionSpec::Tron { + raw_data_hex: raw_tx.raw_data_hex.clone(), + expected_to, + expected_txid: recomputed_txid, + }, + ) } fn encode_protobuf_varint(mut value: u64) -> Vec { @@ -331,8 +335,12 @@ fn take_exact<'a>(input: &mut &'a [u8], length: usize) -> Result<&'a [u8], Strin /// The hand-rolled BIP-32 walk and path parser that used to live here moved /// there wholesale. Custody stays here. fn derive_tron_keypair(mnemonic: &str, derivation_path: &str) -> Result<(Vec, String), String> { - let derived = tinywallet::key::derive(tinywallet::Chain::Tron, mnemonic, derivation_path) - .map_err(|e| e.to_string())?; + let derived = crate::openhuman::web3::wallet::primitives::key::derive( + crate::openhuman::web3::wallet::primitives::Chain::Tron, + mnemonic, + derivation_path, + ) + .map_err(|e| e.to_string())?; Ok(( derived.secret_bytes().to_vec(), derived.address().to_string(), @@ -845,7 +853,7 @@ mod tests { .unwrap(); assert_eq!( native, - tinywallet::wire::TransactionSpec::Tron { + crate::openhuman::web3::wallet::primitives::wire::TransactionSpec::Tron { raw_data_hex: native_raw_hex, expected_to: recipient.to_string(), expected_txid: native_txid, @@ -870,7 +878,7 @@ mod tests { .unwrap(); assert_eq!( token, - tinywallet::wire::TransactionSpec::Tron { + crate::openhuman::web3::wallet::primitives::wire::TransactionSpec::Tron { raw_data_hex: token_raw, expected_to: contract.to_string(), expected_txid: token_txid, diff --git a/src/openhuman/web3/wallet/execution.rs b/src/openhuman/web3/wallet/execution.rs index 6f75155565..d17a7b1062 100644 --- a/src/openhuman/web3/wallet/execution.rs +++ b/src/openhuman/web3/wallet/execution.rs @@ -347,8 +347,8 @@ pub(crate) fn validate_amount(raw: &str) -> Result { /// /// Every arm delegates to the vendored [`tinywallet`] crate, which owns the /// four address formats. The dispatch stays here rather than calling -/// `tinywallet::address::validate` directly because [`WalletChain`] is -/// OpenHuman's enum, and mapping it onto `tinywallet::Chain` here keeps that +/// `crate::openhuman::web3::wallet::primitives::address::validate` directly because [`WalletChain`] is +/// OpenHuman's enum, and mapping it onto `crate::openhuman::web3::wallet::primitives::Chain` here keeps that /// translation in one place. /// /// For Bitcoin this is the **recipient** rule — any well-formed mainnet @@ -357,13 +357,14 @@ pub(crate) fn validate_amount(raw: &str) -> Result { /// the other three chains, so it cannot be expressed through this entry point. fn validate_address(chain: WalletChain, addr: &str) -> Result { let tw_chain = match chain { - WalletChain::Evm => tinywallet::Chain::Evm, - WalletChain::Btc => tinywallet::Chain::Btc, - WalletChain::Solana => tinywallet::Chain::Solana, - WalletChain::Tron => tinywallet::Chain::Tron, + WalletChain::Evm => crate::openhuman::web3::wallet::primitives::Chain::Evm, + WalletChain::Btc => crate::openhuman::web3::wallet::primitives::Chain::Btc, + WalletChain::Solana => crate::openhuman::web3::wallet::primitives::Chain::Solana, + WalletChain::Tron => crate::openhuman::web3::wallet::primitives::Chain::Tron, }; debug!("{LOG_PREFIX} validate_address chain={chain:?} role=recipient dispatch=tinywallet"); - let result = tinywallet::address::validate(tw_chain, addr).map_err(|e| e.to_string()); + let result = crate::openhuman::web3::wallet::primitives::address::validate(tw_chain, addr) + .map_err(|e| e.to_string()); debug!( "{LOG_PREFIX} validate_address chain={chain:?} role=recipient result={}", if result.is_ok() { diff --git a/src/openhuman/web3/wallet/mod.rs b/src/openhuman/web3/wallet/mod.rs index 9f2f61044d..a29ac4469e 100644 --- a/src/openhuman/web3/wallet/mod.rs +++ b/src/openhuman/web3/wallet/mod.rs @@ -33,13 +33,15 @@ mod execution; #[cfg(feature = "web3")] mod ops; #[cfg(feature = "web3")] +pub(crate) mod primitives; +#[cfg(feature = "web3")] pub(crate) mod rpc; #[cfg(feature = "web3")] mod schemas; #[cfg(feature = "web3")] pub mod tools; -/// The host side of `tinywallet`'s `Transport` seam — endpoint resolution, +/// The host side of the wallet primitives' `Transport` seam — endpoint resolution, /// failover and redaction stay here, where the config lives. #[cfg(feature = "web3")] pub(crate) mod transport; diff --git a/src/openhuman/web3/wallet/primitives/abi/mod.rs b/src/openhuman/web3/wallet/primitives/abi/mod.rs new file mode 100644 index 0000000000..3bcca9d992 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/abi/mod.rs @@ -0,0 +1,153 @@ +//! The sliver of Ethereum ABI encoding a wallet actually needs. +//! +//! Exactly one call is encoded here — ERC-20 `transfer(address,uint256)` — and +//! that is deliberate. A general ABI encoder is a parser for a type grammar; a +//! token transfer is a four-byte selector followed by two 32-byte words. Taking +//! a full Ethereum library for the second is how a wallet ends up carrying the +//! first, along with a bignum type and a signer stack. +//! +//! This lives outside the `tx` gate on purpose. Calldata is an *input* to +//! building a transaction, so a host that has moved building into a loadable +//! module still needs to produce it — and would otherwise have to pay a bus +//! round trip for keccak over 68 bytes, or link the chain library it just spent +//! the effort removing. + +use crate::openhuman::web3::wallet::primitives::eip712::u256_from_decimal; + +/// `keccak256("transfer(address,uint256)")[..4]`. +/// +/// Pinned, and re-derived from the signature in the tests below. Every ERC-20 +/// transfer on every EVM chain starts with these four bytes; getting them wrong +/// produces a call that either reverts or, on a contract with a colliding +/// selector, does something else entirely. +const TRANSFER_SELECTOR: [u8; 4] = [0xa9, 0x05, 0x9c, 0xbb]; + +/// The signature the selector is taken from, kept beside it for the test. +#[cfg(test)] +const TRANSFER_SIGNATURE: &[u8] = b"transfer(address,uint256)"; + +/// Why calldata could not be encoded. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The recipient is not a valid EVM address. + #[error("invalid recipient: {reason}")] + InvalidRecipient { + /// What was wrong with it. + reason: String, + }, + + /// The amount is not a base-10 integer, or overflows 256 bits. + #[error("invalid amount: {reason}")] + InvalidAmount { + /// What was wrong with it. + reason: String, + }, +} + +/// Result alias for this module. +pub type Result = std::result::Result; + +/// ABI-encode an ERC-20 `transfer(address,uint256)` call. +/// +/// `amount` is a base-10 string rather than an integer because token amounts +/// are denominated in the token's own smallest unit: an 18-decimal token puts +/// ordinary balances past `u64`, and a caller almost always has the value as +/// text from an RPC or a user. See [`u256_from_decimal`]. +/// +/// Returns `0x`-prefixed hex, which is what `eth_call` and a transaction's +/// `data` field both take. +/// +/// # Errors +/// +/// [`Error::InvalidRecipient`] or [`Error::InvalidAmount`]. +/// +/// # Examples +/// +/// ``` +/// # #[cfg(all(feature = "web3", feature = "web3", feature = "web3"))] { +/// use crate::openhuman::web3::wallet::primitives::abi; +/// +/// let data = abi::encode_erc20_transfer( +/// "0x1111111111111111111111111111111111111111", +/// "1000000", +/// )?; +/// assert!(data.starts_with("0xa9059cbb")); +/// // Selector plus two 32-byte words, hex-encoded, plus the `0x`. +/// assert_eq!(data.len(), 2 + 8 + 128); +/// # } +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::abi::Error>(()) +/// ``` +pub fn encode_erc20_transfer(to: &str, amount: &str) -> Result { + let recipient = crate::openhuman::web3::wallet::primitives::address::evm::validate(to) + .map_err(|e| Error::InvalidRecipient { + reason: e.to_string(), + })?; + let bytes = decode_evm_address(&recipient)?; + let value = u256_from_decimal(amount).map_err(|e| Error::InvalidAmount { + reason: e.to_string(), + })?; + + let mut out = String::with_capacity(2 + 8 + 128); + out.push_str("0x"); + for byte in TRANSFER_SELECTOR { + push_hex(&mut out, byte); + } + // Both arguments are static types, so each is one 32-byte word in order — + // no head/tail offsets, which is the entire reason this can be 20 lines. + for byte in left_pad_address(bytes) { + push_hex(&mut out, byte); + } + for byte in value { + push_hex(&mut out, byte); + } + Ok(out) +} + +/// The 20 raw bytes of an already-validated `0x`-prefixed EVM address. +fn decode_evm_address(address: &str) -> Result<[u8; 20]> { + let body = address.strip_prefix("0x").unwrap_or(address); + let mut out = [0u8; 20]; + for (index, slot) in out.iter_mut().enumerate() { + let pair = body.get(index * 2..index * 2 + 2).ok_or_else(|| { + // Unreachable via `encode_erc20_transfer`, which validates first. + // Mapped rather than unwrapped so a future caller cannot turn a + // malformed address into a panic inside a wallet. + Error::InvalidRecipient { + reason: "address is shorter than 20 bytes".to_string(), + } + })?; + *slot = u8::from_str_radix(pair, 16).map_err(|_| Error::InvalidRecipient { + reason: "address is not hex".to_string(), + })?; + } + Ok(out) +} + +/// An address as the left-padded 32-byte word the ABI encodes it as. +fn left_pad_address(address: [u8; 20]) -> [u8; 32] { + let mut out = [0u8; 32]; + out[12..].copy_from_slice(&address); + out +} + +/// Append one byte as two lowercase hex digits. +fn push_hex(out: &mut String, byte: u8) { + use std::fmt::Write as _; + // Writing into a String cannot fail; discarded rather than unwrapped so + // this stays panic-free. + let _ = write!(out, "{byte:02x}"); +} + +/// Keccak-256, used only by the selector test. +/// +/// Scoped to tests because production code uses the pinned +/// [`TRANSFER_SELECTOR`] rather than hashing the signature on every call. +#[cfg(test)] +fn keccak(bytes: &[u8]) -> [u8; 32] { + use sha3::{Digest as _, Keccak256}; + Keccak256::digest(bytes).into() +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/abi/test.rs b/src/openhuman/web3/wallet/primitives/abi/test.rs new file mode 100644 index 0000000000..da8f087f8c --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/abi/test.rs @@ -0,0 +1,142 @@ +//! Tests for ERC-20 calldata encoding. +//! +//! Calldata is signed, then executed by a contract that will do exactly what +//! the bytes say. A wrong recipient word or a wrong amount word produces a +//! transaction that succeeds and moves the wrong money, so the encoding is +//! checked against the ABI specification's layout rather than against itself. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{encode_erc20_transfer, keccak, Error, TRANSFER_SELECTOR, TRANSFER_SIGNATURE}; + +const RECIPIENT: &str = "0x1111111111111111111111111111111111111111"; + +#[test] +fn the_pinned_selector_matches_its_signature() { + // The constant is pinned so the hash is not recomputed per call; this is + // the test that makes pinning safe rather than a place for a typo to hide. + assert_eq!(keccak(TRANSFER_SIGNATURE)[..4], TRANSFER_SELECTOR); +} + +#[test] +fn the_selector_is_the_published_erc20_transfer_selector() { + assert_eq!(TRANSFER_SELECTOR, [0xa9, 0x05, 0x9c, 0xbb]); +} + +#[test] +fn the_encoding_is_a_selector_and_two_left_padded_words() { + let data = encode_erc20_transfer(RECIPIENT, "1000000").unwrap(); + + // 0x + 4-byte selector + 2 x 32-byte words, hex. + assert_eq!(data.len(), 2 + 8 + 128); + assert_eq!( + data, + "0xa9059cbb\ + 0000000000000000000000001111111111111111111111111111111111111111\ + 00000000000000000000000000000000000000000000000000000000000f4240" + ); +} + +#[test] +fn the_recipient_is_right_aligned_in_its_word() { + // Left-padding is the ABI rule for `address`. Getting it backwards yields + // a well-formed call paying an address nobody controls. + let data = encode_erc20_transfer(RECIPIENT, "1").unwrap(); + let recipient_word = &data[10..74]; + assert!(recipient_word.starts_with(&"0".repeat(24))); + assert!(recipient_word.ends_with(&"11".repeat(20))); +} + +#[test] +fn an_amount_beyond_u64_encodes_exactly() { + // The reason the amount is a string: an 18-decimal token puts ordinary + // balances past u64, and truncating would silently transfer the wrong sum. + let data = encode_erc20_transfer(RECIPIENT, "340282366920938463463374607431768211456").unwrap(); + assert!(data.ends_with("0000000000000000000000000000000100000000000000000000000000000000")); +} + +#[test] +fn the_largest_representable_amount_is_accepted() { + let max = "115792089237316195423570985008687907853269984665640564039457584007913129639935"; + let data = encode_erc20_transfer(RECIPIENT, max).unwrap(); + assert!(data.ends_with(&"f".repeat(64))); +} + +#[test] +fn a_zero_amount_encodes_as_a_zero_word_not_an_empty_one() { + // Static types are always a full word; an empty encoding would shift the + // call's shape and make it unparseable by the contract. + let data = encode_erc20_transfer(RECIPIENT, "0").unwrap(); + assert_eq!(data.len(), 2 + 8 + 128); + assert!(data.ends_with(&"0".repeat(64))); +} + +#[test] +fn a_checksummed_recipient_encodes_the_same_as_a_lowercase_one() { + // EIP-55 casing is display metadata, not part of the address. + let lower = encode_erc20_transfer("0xab5801a7d398351b8be11c439e05c5b3259aec9b", "5").unwrap(); + let checksummed = + encode_erc20_transfer("0xAb5801a7D398351b8bE11C439e05C5B3259aeC9B", "5").unwrap(); + assert_eq!(lower, checksummed); +} + +#[test] +fn an_invalid_recipient_is_refused() { + for bad in [ + "", + "0x", + "not-an-address", + "0x111", + &format!("0x{}", "1".repeat(41)), + ] { + assert!( + matches!( + encode_erc20_transfer(bad, "1"), + Err(Error::InvalidRecipient { .. }) + ), + "{bad:?} should be refused" + ); + } +} + +#[test] +fn a_non_numeric_or_overflowing_amount_is_refused() { + for bad in [ + "", + "12a", + "-1", + "1.5", + "0x10", + // 2^256 exactly: one past the top. + "115792089237316195423570985008687907853269984665640564039457584007913129639936", + ] { + assert!( + matches!( + encode_erc20_transfer(RECIPIENT, bad), + Err(Error::InvalidAmount { .. }) + ), + "{bad:?} should be refused" + ); + } +} + +#[test] +fn the_address_decoder_refuses_malformed_input_rather_than_panicking() { + // `encode_erc20_transfer` validates before calling this, so these arms are + // defensive — but defensive code that is never exercised is code nobody + // knows works, and the failure mode it guards against is a panic inside a + // wallet. Tested directly because the public path cannot reach it. + use super::decode_evm_address; + + assert!(matches!( + decode_evm_address("0x1111"), + Err(Error::InvalidRecipient { .. }) + )); + assert!(matches!( + decode_evm_address(&format!("0x{}", "zz".repeat(20))), + Err(Error::InvalidRecipient { .. }) + )); + + // The happy path, unprefixed, to pin that the `0x` is optional here. + assert_eq!(decode_evm_address(&"11".repeat(20)).unwrap(), [0x11u8; 20]); +} diff --git a/src/openhuman/web3/wallet/primitives/address/btc.rs b/src/openhuman/web3/wallet/primitives/address/btc.rs new file mode 100644 index 0000000000..74d806e424 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/btc.rs @@ -0,0 +1,292 @@ +//! Bitcoin address validation. +//! +//! Two functions, because Bitcoin has two different answers depending on which +//! side of a transaction the address sits on: +//! +//! - [`validate`] — any well-formed mainnet address. Correct for a +//! **recipient**: we do not care which address type they prefer, because +//! paying to a P2WPKH, P2TR, P2SH or P2PKH output is the same operation. +//! - [`validate_sender`] — additionally requires **P2WPKH** (`bc1q…` native +//! segwit). Correct for a **sender**, because that is the only script type +//! this crate's family of signing paths knows how to spend. +//! +//! Calling [`validate`] where [`validate_sender`] belongs is the dangerous +//! direction: it accepts an address that will fail much later, at signing +//! time, after a transaction has been assembled. The two are separate +//! functions rather than a boolean flag so that mistake reads wrong at the +//! call site. +//! +//! # Why this does not use the `bitcoin` crate +//! +//! It used to. The crate is excellent and this module is a strictly smaller +//! thing than what it offers — but it carries `secp256k1`, and therefore a +//! native C build, into every consumer that only ever wanted to check whether a +//! string is a well-formed address. That cost is invisible in a full wallet and +//! dominant in a host that has moved signing elsewhere. +//! +//! Address *parsing* is a safe thing to own directly, unlike the BIP-32 walk in +//! [`crate::openhuman::web3::wallet::primitives::key`], which deliberately still delegates. The distinction is +//! failure mode, not difficulty: a parser that is wrong rejects a good address +//! or accepts a malformed one, and both are caught immediately by the vectors +//! below. A derivation that is wrong returns a *valid key for the wrong +//! account* — silently, and unrecoverably. So this module is hand-rolled +//! against the published BIP-173 and BIP-350 vectors, and key derivation is +//! not. +//! +//! The five mainnet forms, in full: +//! +//! | Type | Encoding | Prefix / witness version | Program length | +//! | --- | --- | --- | --- | +//! | P2PKH | base58check | version byte `0x00` | 20 | +//! | P2SH | base58check | version byte `0x05` | 20 | +//! | P2WPKH | bech32 | `bc`, v0 | 20 | +//! | P2WSH | bech32 | `bc`, v0 | 32 | +//! | P2TR | bech32m | `bc`, v1 | 32 | +//! +//! Witness versions 2..=16 are accepted as recipients with a 2..=40 byte +//! program, per BIP-350. Refusing them would make this crate reject addresses +//! that are valid today and spendable by their owners, purely because a future +//! output type had not been invented when it was written. + +use crate::openhuman::web3::wallet::primitives::chain::Chain; +use crate::openhuman::web3::wallet::primitives::{Error, Result}; + +/// Human-readable part of a Bitcoin **mainnet** bech32 address. +const MAINNET_HRP: &str = "bc"; + +/// Base58check version byte for P2PKH. +const P2PKH_VERSION: u8 = 0x00; + +/// Base58check version byte for P2SH. +const P2SH_VERSION: u8 = 0x05; + +/// Base58check version bytes belonging to Bitcoin test networks. +const TEST_VERSIONS: [u8; 2] = [0x6f, 0xc4]; + +/// What a well-formed mainnet address turned out to be. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + /// Pay to public key hash — legacy, base58. + P2pkh, + /// Pay to script hash — base58. + P2sh, + /// Pay to witness public key hash — the only spendable-from type here. + P2wpkh, + /// Pay to witness script hash. + P2wsh, + /// A segwit output that is none of the above: taproot, or a future version. + OtherWitness, +} + +/// Validate a Bitcoin **mainnet** address of any type, returning it trimmed. +/// +/// Use this for transaction recipients. +/// +/// # Errors +/// +/// - [`Error::EmptyAddress`] if `address` is empty or all whitespace. +/// - [`Error::InvalidAddress`] if it does not parse as a Bitcoin address. +/// - [`Error::WrongNetwork`] if it parses but belongs to testnet, signet, or +/// regtest. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::btc; +/// +/// // Native segwit, wrapped segwit, legacy, and taproot are all accepted. +/// assert!(btc::validate("bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4").is_ok()); +/// assert!(btc::validate("1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2").is_ok()); +/// +/// // A testnet address is well-formed but on the wrong network. +/// assert!(btc::validate("tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx").is_err()); +/// ``` +pub fn validate(address: &str) -> Result { + let trimmed = trimmed_non_empty(address)?; + parse(trimmed)?; + Ok(trimmed.to_string()) +} + +/// Validate a Bitcoin address usable as a **sender**, returning it trimmed. +/// +/// Everything [`validate`] requires, plus the address must be P2WPKH — native +/// segwit, the `bc1q…` form. Signing is only implemented for that script type, +/// so any other type would fail later with a much less obvious error. +/// +/// # Errors +/// +/// - Everything [`validate`] returns. +/// - [`Error::UnsupportedAddressType`] if the address is well-formed mainnet +/// but not P2WPKH. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::btc; +/// +/// // Native segwit: usable as a sender. +/// assert!(btc::validate_sender("bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4").is_ok()); +/// +/// // A legacy address is a fine recipient but cannot be signed for here. +/// assert!(btc::validate("1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2").is_ok()); +/// assert!(btc::validate_sender("1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2").is_err()); +/// ``` +pub fn validate_sender(address: &str) -> Result { + let trimmed = trimmed_non_empty(address)?; + // Deliberately ordered: a malformed or wrong-network address is reported as + // such, never as an unsupported *type*, which would point at the wrong fix. + if parse(trimmed)? != Kind::P2wpkh { + return Err(Error::UnsupportedAddressType { + chain: Chain::Btc, + address: trimmed.to_string(), + reason: "only P2WPKH (bc1q… native segwit) can be signed for".to_string(), + }); + } + Ok(trimmed.to_string()) +} + +/// Encode a 20-byte public key hash as a mainnet P2WPKH (`bc1q…`) address. +/// +/// The counterpart to parsing: [`crate::openhuman::web3::wallet::primitives::key`] derives a public key and needs +/// its address, and doing that here keeps the bech32 encoding in the module +/// that also decodes it. +/// +/// # Errors +/// +/// [`Error::InvalidAddress`] only if bech32 encoding fails, which for a +/// fixed-length v0 program and a constant HRP it cannot. +pub(crate) fn encode_p2wpkh(pubkey_hash: &[u8; 20]) -> Result { + // `hrp::BC` rather than parsing `MAINNET_HRP`: the parse could not fail for + // a two-letter constant, and an error arm that cannot fire is one nothing + // can test. + bech32::segwit::encode_v0(bech32::hrp::BC, pubkey_hash).map_err(|e| Error::InvalidAddress { + chain: Chain::Btc, + address: String::new(), + reason: e.to_string(), + }) +} + +/// Trim `address` and reject it if nothing is left. +fn trimmed_non_empty(address: &str) -> Result<&str> { + let trimmed = address.trim(); + if trimmed.is_empty() { + return Err(Error::EmptyAddress { chain: Chain::Btc }); + } + Ok(trimmed) +} + +/// Identify a mainnet address, or say why it is not one. +/// +/// Dispatch is on *shape*, not on a list of known prefixes. A bech32 string is +/// an all-letter human-readable part, a `1` separator, then a data part drawn +/// from an alphabet that excludes `1` — so the last `1` is the separator, and +/// what precedes it is the HRP. +/// +/// Routing every bech32-shaped string to [`parse_bech32`], rather than only +/// those starting `bc1`, is what lets a testnet or foreign-chain address be +/// reported as the wrong network instead of as malformed base58. Matching on a +/// hardcoded prefix list left that check unreachable and gave a Litecoin +/// address a base58 error message. +fn parse(address: &str) -> Result { + let lower = address.to_ascii_lowercase(); + if let Some(separator) = lower.rfind('1') { + let hrp = &lower[..separator]; + if !hrp.is_empty() && hrp.chars().all(|c| c.is_ascii_lowercase()) { + return parse_bech32(address); + } + } + parse_base58(address) +} + +/// Decode a bech32 or bech32m segwit address. +/// +/// One call does the whole job: `bech32::segwit::decode` rejects a witness +/// version above 16, selects the checksum algorithm the version requires +/// (bech32 for v0, bech32m for v1+, per BIP-350), rejects mixed case, and +/// enforces the program-length rules — 20 or 32 bytes at v0, 2..=40 above it. +/// Re-checking any of that here would be a second, drifting implementation of +/// rules the crate already owns. +fn parse_bech32(address: &str) -> Result { + let (hrp, version, program) = + bech32::segwit::decode(address).map_err(|e| Error::InvalidAddress { + chain: Chain::Btc, + address: address.to_string(), + reason: e.to_string(), + })?; + + if hrp.as_str() != MAINNET_HRP { + return Err(wrong_network(address, "a non-mainnet human-readable part")); + } + + // Only v0 needs discriminating, because only P2WPKH is spendable here. + // Taproot and every future version are payable recipients and nothing more, + // so they share one arm rather than each earning a variant that no caller + // would branch on. + if version.to_u8() != 0 { + return Ok(Kind::OtherWitness); + } + match program.len() { + 20 => Ok(Kind::P2wpkh), + // Guaranteed 32 by the length validation above; spelled out rather than + // wildcarded so a future relaxation upstream cannot silently land here + // as "P2WSH". + 32 => Ok(Kind::P2wsh), + other => Err(Error::InvalidAddress { + chain: Chain::Btc, + address: address.to_string(), + reason: format!("witness v0 program must be 20 or 32 bytes, got {other}"), + }), + } +} + +/// Decode a base58check P2PKH or P2SH address. +fn parse_base58(address: &str) -> Result { + let decoded = bs58::decode(address) + .with_check(None) + .into_vec() + .map_err(|e| Error::InvalidAddress { + chain: Chain::Btc, + address: address.to_string(), + reason: e.to_string(), + })?; + + // base58check strips the 4-byte checksum, leaving version || payload. + let (version, payload) = decoded.split_first().ok_or_else(|| Error::InvalidAddress { + chain: Chain::Btc, + address: address.to_string(), + reason: "empty base58check payload".to_string(), + })?; + + if TEST_VERSIONS.contains(version) { + return Err(wrong_network(address, "a test network version byte")); + } + if payload.len() != 20 { + return Err(Error::InvalidAddress { + chain: Chain::Btc, + address: address.to_string(), + reason: format!("hash must be 20 bytes, got {}", payload.len()), + }); + } + match *version { + P2PKH_VERSION => Ok(Kind::P2pkh), + P2SH_VERSION => Ok(Kind::P2sh), + other => Err(Error::InvalidAddress { + chain: Chain::Btc, + address: address.to_string(), + reason: format!("unknown base58check version byte {other:#04x}"), + }), + } +} + +/// A well-formed address that belongs to another network. +fn wrong_network(address: &str, reason: &str) -> Error { + Error::WrongNetwork { + chain: Chain::Btc, + address: address.to_string(), + expected: "mainnet".to_string(), + reason: reason.to_string(), + } +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/address/btc/test.rs b/src/openhuman/web3/wallet/primitives/address/btc/test.rs new file mode 100644 index 0000000000..2170d113c6 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/btc/test.rs @@ -0,0 +1,288 @@ +//! Unit tests for Bitcoin address validation. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{validate, validate_sender}; +use crate::openhuman::web3::wallet::primitives::{Chain, Error}; + +/// P2WPKH — native segwit. The only type valid as a sender. +const P2WPKH: &str = "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4"; +/// P2PKH — legacy. A fine recipient. +const P2PKH: &str = "1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2"; +/// P2SH — wrapped segwit or multisig. A fine recipient. +const P2SH: &str = "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy"; +/// P2WSH — native segwit script hash. A fine recipient. +const P2WSH: &str = "bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3"; + +#[test] +fn accepts_every_mainnet_address_type_as_a_recipient() { + for addr in [P2WPKH, P2PKH, P2SH, P2WSH] { + assert!(validate(addr).is_ok(), "{addr} should validate"); + } +} + +#[test] +fn trims_surrounding_whitespace() { + assert_eq!(validate(&format!(" {P2WPKH}\n")).unwrap(), P2WPKH); +} + +#[test] +fn rejects_an_empty_address() { + assert_eq!( + validate(" ").unwrap_err(), + Error::EmptyAddress { chain: Chain::Btc } + ); +} + +#[test] +fn rejects_a_malformed_address() { + assert!(matches!( + validate("not-an-address").unwrap_err(), + Error::InvalidAddress { .. } + )); +} + +#[test] +fn rejects_a_mistyped_address_via_its_checksum() { + // Bitcoin addresses are checksummed, so a single changed character is + // caught rather than naming a different account. + let mut chars: Vec = P2PKH.chars().collect(); + chars[5] = if chars[5] == 'a' { 'b' } else { 'a' }; + let typo: String = chars.into_iter().collect(); + assert_ne!(typo, P2PKH, "the fixture must actually differ"); + assert!( + validate(&typo).is_err(), + "a checksum failure must be caught" + ); +} + +#[test] +fn rejects_a_testnet_address_as_the_wrong_network() { + // Well-formed, but on the wrong network — a distinct variant because it is + // the failure a caller is likely to handle rather than merely report. + let testnet = "tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx"; + match validate(testnet).unwrap_err() { + Error::WrongNetwork { + chain, + address, + expected, + .. + } => { + assert_eq!(chain, Chain::Btc); + assert_eq!(address, testnet); + assert_eq!(expected, "mainnet"); + } + other => panic!("expected WrongNetwork, got {other:?}"), + } +} + +#[test] +fn accepts_p2wpkh_as_a_sender() { + assert_eq!(validate_sender(P2WPKH).unwrap(), P2WPKH); +} + +#[test] +fn rejects_every_non_p2wpkh_type_as_a_sender() { + // These are all valid recipients. The sender rule is strictly narrower + // because signing is only implemented for P2WPKH. + for addr in [P2PKH, P2SH, P2WSH] { + assert!( + validate(addr).is_ok(), + "{addr} must remain a valid recipient" + ); + match validate_sender(addr).unwrap_err() { + Error::UnsupportedAddressType { chain, address, .. } => { + assert_eq!(chain, Chain::Btc); + assert_eq!(address, addr); + } + other => panic!("expected UnsupportedAddressType for {addr}, got {other:?}"), + } + } +} + +#[test] +fn sender_validation_still_reports_the_underlying_failure_first() { + // A malformed or wrong-network address should not be reported as an + // unsupported *type* — that would point at the wrong fix. + assert!(matches!( + validate_sender("garbage").unwrap_err(), + Error::InvalidAddress { .. } + )); + assert!(matches!( + validate_sender("tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx").unwrap_err(), + Error::WrongNetwork { .. } + )); + assert!(matches!( + validate_sender(" ").unwrap_err(), + Error::EmptyAddress { .. } + )); +} + +// --------------------------------------------------------------------------- +// Branch coverage for the hand-rolled parser. +// +// These are the paths that only exist because this module stopped delegating +// to the `bitcoin` crate. Each one is a rejection, and a rejection that never +// fires is indistinguishable from one that is wrong — so every arm gets a +// vector, drawn from BIP-173 and BIP-350 where they publish one. +// --------------------------------------------------------------------------- + +/// P2TR — taproot, witness v1, bech32m. A valid recipient, not a sender. +const P2TR: &str = "bc1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqzk5jj0"; + +#[test] +fn accepts_taproot_as_a_recipient_but_not_as_a_sender() { + // Witness v1 uses bech32m rather than bech32; accepting it proves the + // checksum variant is selected by version rather than assumed. + assert_eq!(validate(P2TR).unwrap(), P2TR); + assert!(matches!( + validate_sender(P2TR).unwrap_err(), + Error::UnsupportedAddressType { .. } + )); +} + +#[test] +fn rejects_a_v0_address_carrying_a_bech32m_checksum() { + // BIP-350's central rule. Both strings below are well-formed bech32-ish; + // what separates them is which checksum constant they were built with, and + // accepting the wrong one would accept addresses no other wallet does. + let v0_with_bech32m = "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kemeawh"; + assert!(validate(v0_with_bech32m).is_err()); +} + +#[test] +fn rejects_a_taproot_address_carrying_a_bech32_checksum() { + // The mirror of the case above: v1 must be bech32m. + let v1_with_bech32 = "bc1p38j9r5y49hruaue7wxjce0updqjuyyx0kh56v8s25huc6995vvpql3jow4"; + assert!(validate(v1_with_bech32).is_err()); +} + +#[test] +fn rejects_a_witness_program_with_the_wrong_checksum_for_version_three() { + // BIP-350: witness versions 1..=16 require bech32m, not bech32. + let v0_16_bytes = "bc1rw5uspcuh"; + assert!(validate(v0_16_bytes).is_err()); +} + +#[test] +fn rejects_a_mixed_case_bech32_address() { + // Mixed case is invalid per BIP-173 because it breaks the checksum's + // case-folding guarantee. + let mixed = "bc1QW508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4"; + assert!(validate(mixed).is_err()); +} + +#[test] +fn reports_a_testnet_base58_address_as_the_wrong_network_not_as_malformed() { + // A testnet P2PKH is perfectly well-formed; naming it correctly is the + // difference between a user fixing their address and thinking it is broken. + let testnet_p2pkh = "mipcBbFg9gMiCh81Kj8tqqdgoZub1ZJRfn"; + assert!(matches!( + validate(testnet_p2pkh).unwrap_err(), + Error::WrongNetwork { .. } + )); +} + +#[test] +fn reports_a_regtest_bech32_address_as_the_wrong_network() { + let regtest = "bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080"; + assert!(matches!( + validate(regtest).unwrap_err(), + Error::WrongNetwork { .. } + )); +} + +#[test] +fn rejects_a_base58_address_with_an_unknown_version_byte() { + // Valid base58check, valid length, but a version byte that is neither + // P2PKH nor P2SH on mainnet — a namecoin address, for instance. + let unknown_version = "NCXn6ZQTr8GN5T4bB1oSHnLRcNPQXswcpv"; + match validate(unknown_version) { + Err(Error::InvalidAddress { .. } | Error::WrongNetwork { .. }) => {} + other => panic!("expected a rejection, got {other:?}"), + } +} + +#[test] +fn rejects_a_bech32_address_for_another_coin() { + // Well-formed bech32 with a human-readable part that is not Bitcoin's. + let not_bitcoin = "ltc1qw508d6qejxtdg4y5r3zarvary0c5xw7kgmn4n9"; + assert!(validate(not_bitcoin).is_err()); +} + +#[test] +fn encodes_a_p2wpkh_address_its_own_validator_accepts() { + // Closes the loop: what `key::btc` produces must parse back here, and the + // encoder is the only part of this module the validators do not exercise. + let pubkey_hash = [0x75u8; 20]; + let encoded = super::encode_p2wpkh(&pubkey_hash).unwrap(); + assert!(encoded.starts_with("bc1q")); + assert_eq!(validate_sender(&encoded).unwrap(), encoded); +} + +/// Encode `version || payload` as base58check, the way a real address is built. +/// +/// Constructed rather than copied from a block explorer because these vectors +/// have to be *valid* base58check that is wrong in one specific way — a +/// hand-typed string would fail its checksum first and never reach the rule +/// under test. +fn base58check(version: u8, payload: &[u8]) -> String { + let mut body = Vec::with_capacity(1 + payload.len()); + body.push(version); + body.extend_from_slice(payload); + bs58::encode(body).with_check().into_string() +} + +#[test] +fn rejects_a_base58_address_with_an_unrecognised_version_byte() { + // Valid checksum, 20-byte hash, but a version that is neither P2PKH (0x00) + // nor P2SH (0x05) on mainnet — a Litecoin P2PKH, for instance. + let litecoin = base58check(0x30, &[0x11; 20]); + match validate(&litecoin).unwrap_err() { + Error::InvalidAddress { reason, .. } => assert!(reason.contains("version"), "{reason}"), + other => panic!("expected InvalidAddress, got {other:?}"), + } +} + +#[test] +fn rejects_a_base58_address_whose_hash_is_the_wrong_length() { + // A well-formed base58check envelope around a 19-byte hash. Accepting it + // would build a transaction paying a script nobody can spend. + let short = base58check(0x00, &[0x11; 19]); + match validate(&short).unwrap_err() { + Error::InvalidAddress { reason, .. } => assert!(reason.contains("20 bytes"), "{reason}"), + other => panic!("expected InvalidAddress, got {other:?}"), + } +} + +#[test] +fn rejects_an_empty_base58check_payload() { + // Checksum over nothing at all: there is no version byte to read. + let empty = bs58::encode(Vec::::new()).with_check().into_string(); + assert!(validate(&empty).is_err()); +} + +#[test] +fn reports_a_testnet_p2sh_version_as_the_wrong_network() { + // 0xc4 is testnet P2SH. The sibling 0x6f (testnet P2PKH) is covered above + // by a real address; this one completes the pair. + let testnet_p2sh = base58check(0xc4, &[0x11; 20]); + assert!(matches!( + validate(&testnet_p2sh).unwrap_err(), + Error::WrongNetwork { .. } + )); +} + +#[test] +fn reports_a_foreign_bech32_chain_as_the_wrong_network_not_as_bad_base58() { + // The check this exercises was unreachable when dispatch matched on a + // hardcoded `bc1` prefix: a Litecoin bech32 address fell through to the + // base58 parser and came back with a nonsensical error. + let litecoin = "ltc1qw508d6qejxtdg4y5r3zarvary0c5xw7kgmn4n9"; + match validate(litecoin).unwrap_err() { + Error::WrongNetwork { reason, .. } => { + assert!(reason.contains("human-readable part"), "{reason}"); + } + other => panic!("expected WrongNetwork, got {other:?}"), + } +} diff --git a/src/openhuman/web3/wallet/primitives/address/evm.rs b/src/openhuman/web3/wallet/primitives/address/evm.rs new file mode 100644 index 0000000000..27849c2686 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/evm.rs @@ -0,0 +1,177 @@ +//! EVM address validation. +//! +//! An EVM address is 20 bytes rendered as 40 hex digits, conventionally with a +//! `0x` prefix. That is the entire format, which is why this module has no +//! dependencies: pulling a chain client in to call its `Address::from_str` +//! would drag a large secp256k1 and RLP stack in to check a string is hex. +//! +//! ## EIP-55 is checked separately, and that is deliberate +//! +//! [`validate`] accepts any correctly-shaped address, mixed case included, +//! without verifying an EIP-55 checksum. Rejecting a non-checksummed address +//! would break every lowercase address in the wild — they are valid, just +//! unchecksummed, and most tooling emits them. +//! +//! [`is_checksum_valid`] is offered separately for a caller that *has* a +//! mixed-case address and wants the typo protection EIP-55 provides. Keeping +//! the two apart means a host chooses its own strictness instead of inheriting +//! ours. + +use crate::openhuman::web3::wallet::primitives::chain::Chain; +use crate::openhuman::web3::wallet::primitives::{Error, Result}; + +/// Number of hex digits in an EVM address: 20 bytes, two digits each. +const ADDRESS_HEX_LEN: usize = 40; + +/// Validate an EVM address and return it trimmed. +/// +/// Accepts an address with or without the lowercase `0x` prefix. The hex body +/// may be any case; an uppercase `0X` prefix is **rejected**, since no tooling +/// emits it and accepting it would widen what counts as an address for no +/// benefit. The returned string is the input with surrounding whitespace +/// removed and is +/// otherwise **unmodified** — case and prefix are preserved, because callers +/// echo the address back to users and normalising it would silently change +/// what they typed. Use [`to_checksummed`] when a canonical form is wanted. +/// +/// # Errors +/// +/// - [`Error::EmptyAddress`] if `address` is empty or all whitespace. +/// - [`Error::InvalidAddress`] if it is not 40 hex digits after an optional +/// `0x` prefix. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::evm; +/// +/// let addr = evm::validate(" 0x52908400098527886E0F7030069857D2E4169EE7 ")?; +/// assert_eq!(addr, "0x52908400098527886E0F7030069857D2E4169EE7"); +/// +/// // The `0x` prefix is optional. +/// assert!(evm::validate("52908400098527886E0F7030069857D2E4169EE7").is_ok()); +/// +/// // But an uppercase `0X` prefix is not accepted. +/// assert!(evm::validate("0X52908400098527886E0F7030069857D2E4169EE7").is_err()); +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::Error>(()) +/// ``` +pub fn validate(address: &str) -> Result { + let trimmed = address.trim(); + if trimmed.is_empty() { + return Err(Error::EmptyAddress { chain: Chain::Evm }); + } + + let body = strip_prefix(trimmed); + if body.len() != ADDRESS_HEX_LEN { + return Err(Error::InvalidAddress { + chain: Chain::Evm, + address: trimmed.to_string(), + reason: format!("expected {ADDRESS_HEX_LEN} hex digits, got {}", body.len()), + }); + } + if let Some(bad) = body.chars().find(|c| !c.is_ascii_hexdigit()) { + return Err(Error::InvalidAddress { + chain: Chain::Evm, + address: trimmed.to_string(), + reason: format!("contains a non-hex character '{bad}'"), + }); + } + Ok(trimmed.to_string()) +} + +/// Strip an optional lowercase `0x` prefix. +/// +/// Deliberately does not accept `0X`: see [`validate`]. +fn strip_prefix(address: &str) -> &str { + address.strip_prefix("0x").unwrap_or(address) +} + +/// Whether `address` carries a valid EIP-55 checksum. +/// +/// This compares the input against the canonical mixed-case rendering +/// [`to_checksummed`] produces: it returns `true` only when the two are +/// byte-for-byte identical. That is what makes a typo detectable — a wrong +/// character almost always breaks the agreement. +/// +/// Concretely, an all-uppercase address never matches, and an all-lowercase +/// one usually does not either, because the canonical form is normally +/// mixed-case. The `usually` matters: an address whose canonical form is +/// itself entirely lowercase (as with +/// `0xde709f2102306220921060314715629080e2fb77`) matches as-is. So this +/// answers "does the address carry a correct checksum", not "is it a valid +/// address" — pair it with [`validate`], which is the function that answers +/// the latter. +/// +/// # Errors +/// +/// Propagates [`validate`]'s errors: the address must be well-formed before a +/// checksum question is meaningful. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::evm; +/// +/// // A correctly checksummed address. +/// assert!(evm::is_checksum_valid("0x52908400098527886E0F7030069857D2E4169EE7")?); +/// // Valid, but carries no checksum information. +/// assert!(!evm::is_checksum_valid("0x52908400098527886e0f7030069857d2e4169ee7")?); +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::Error>(()) +/// ``` +#[cfg(feature = "web3")] +pub fn is_checksum_valid(address: &str) -> Result { + let validated = validate(address)?; + Ok(to_checksummed(&validated)? == prefixed(strip_prefix(&validated))) +} + +/// Render `address` in canonical EIP-55 mixed-case form, `0x`-prefixed. +/// +/// # Errors +/// +/// Propagates [`validate`]'s errors. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::evm; +/// +/// let canonical = evm::to_checksummed("0x52908400098527886e0f7030069857d2e4169ee7")?; +/// assert_eq!(canonical, "0x52908400098527886E0F7030069857D2E4169EE7"); +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::Error>(()) +/// ``` +#[cfg(feature = "web3")] +pub fn to_checksummed(address: &str) -> Result { + use sha3::{Digest, Keccak256}; + + let validated = validate(address)?; + let lower = strip_prefix(&validated).to_ascii_lowercase(); + let hash = Keccak256::digest(lower.as_bytes()); + + let mut out = String::with_capacity(2 + ADDRESS_HEX_LEN); + out.push_str("0x"); + for (i, c) in lower.chars().enumerate() { + // EIP-55: uppercase digit `i` when nibble `i` of the hash is >= 8. + // Nibbles are big-endian within each byte, so even indices take the + // high nibble. + let nibble = if i % 2 == 0 { + hash[i / 2] >> 4 + } else { + hash[i / 2] & 0x0f + }; + if c.is_ascii_digit() || nibble < 8 { + out.push(c); + } else { + out.push(c.to_ascii_uppercase()); + } + } + Ok(out) +} + +/// Re-attach the `0x` prefix to a bare hex body. +#[cfg(feature = "web3")] +fn prefixed(body: &str) -> String { + format!("0x{body}") +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/address/evm/test.rs b/src/openhuman/web3/wallet/primitives/address/evm/test.rs new file mode 100644 index 0000000000..b6684a035e --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/evm/test.rs @@ -0,0 +1,183 @@ +//! Unit tests for EVM address validation. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::validate; +use crate::openhuman::web3::wallet::primitives::{Chain, Error}; + +/// The four EIP-55 test vectors from the specification. +const EIP55_VECTORS: [&str; 4] = [ + "0x52908400098527886E0F7030069857D2E4169EE7", + "0x8617E340B3D01FA5F11F306F4090FD50E238070D", + "0xde709f2102306220921060314715629080e2fb77", + "0x27b1fdb04752bbc536007a920d24acb045561c26", +]; + +#[test] +fn accepts_a_prefixed_address() { + assert_eq!( + validate("0x52908400098527886E0F7030069857D2E4169EE7").unwrap(), + "0x52908400098527886E0F7030069857D2E4169EE7" + ); +} + +#[test] +fn accepts_an_unprefixed_address() { + let bare = "52908400098527886E0F7030069857D2E4169EE7"; + assert_eq!(validate(bare).unwrap(), bare); +} + +#[test] +fn rejects_an_uppercase_prefix() { + // No tooling emits `0X`, so accepting it would widen what counts as an + // address for no benefit. It falls out as a length failure: `0X…` is 42 + // characters once the prefix is not stripped. + assert!(matches!( + validate("0X52908400098527886E0F7030069857D2E4169EE7").unwrap_err(), + Error::InvalidAddress { .. } + )); +} + +#[test] +fn preserves_the_input_rather_than_normalising_it() { + // Callers echo the returned address back to users, so changing its case or + // stripping its prefix would silently alter what they typed. + let lower = "0x52908400098527886e0f7030069857d2e4169ee7"; + assert_eq!(validate(lower).unwrap(), lower); +} + +#[test] +fn trims_surrounding_whitespace() { + assert_eq!( + validate(" 0x52908400098527886E0F7030069857D2E4169EE7\n").unwrap(), + "0x52908400098527886E0F7030069857D2E4169EE7" + ); +} + +#[test] +fn rejects_an_empty_address() { + assert_eq!( + validate(" ").unwrap_err(), + Error::EmptyAddress { chain: Chain::Evm } + ); +} + +#[test] +fn rejects_a_short_address() { + match validate("0xdeadbeef").unwrap_err() { + Error::InvalidAddress { chain, reason, .. } => { + assert_eq!(chain, Chain::Evm); + assert!(reason.contains("got 8"), "reason was {reason:?}"); + } + other => panic!("expected InvalidAddress, got {other:?}"), + } +} + +#[test] +fn rejects_a_long_address() { + let long = format!("0x{}", "a".repeat(41)); + assert!(matches!( + validate(&long).unwrap_err(), + Error::InvalidAddress { .. } + )); +} + +#[test] +fn rejects_a_non_hex_character() { + // 40 characters, but `z` is not hex — a length check alone would pass it. + let bad = format!("0x{}z", "a".repeat(39)); + match validate(&bad).unwrap_err() { + Error::InvalidAddress { reason, .. } => { + assert!( + reason.contains('z'), + "reason should name the char: {reason:?}" + ); + } + other => panic!("expected InvalidAddress, got {other:?}"), + } +} + +#[test] +fn the_error_carries_the_rejected_address_verbatim() { + // Diagnosing a rejection means seeing exactly what was rejected. + match validate("0xnope").unwrap_err() { + Error::InvalidAddress { address, .. } => assert_eq!(address, "0xnope"), + other => panic!("expected InvalidAddress, got {other:?}"), + } +} + +#[test] +fn accepts_every_eip55_vector_regardless_of_case() { + // Validation is case-insensitive: an unchecksummed lowercase address is + // valid, just unchecksummed. + for vector in EIP55_VECTORS { + assert!(validate(vector).is_ok(), "{vector} should validate"); + assert!(validate(&vector.to_lowercase()).is_ok()); + } +} + +#[cfg(feature = "web3")] +mod checksum { + use super::EIP55_VECTORS; + use crate::openhuman::web3::wallet::primitives::address::evm::{ + is_checksum_valid, to_checksummed, + }; + + #[test] + fn canonicalises_every_eip55_vector() { + for vector in EIP55_VECTORS { + assert_eq!( + to_checksummed(&vector.to_lowercase()).unwrap(), + *vector, + "EIP-55 vector {vector} did not round-trip" + ); + } + } + + #[test] + fn accepts_a_correctly_checksummed_address() { + for vector in EIP55_VECTORS { + assert!(is_checksum_valid(vector).unwrap(), "{vector}"); + } + } + + #[test] + fn rejects_a_wrongly_cased_address() { + // Flip the case of one letter in a checksummed vector. + let vector = "0x52908400098527886E0F7030069857D2E4169EE7"; + let broken = vector.replacen('E', "e", 1); + assert_ne!(broken, vector, "the fixture must actually differ"); + assert!(!is_checksum_valid(&broken).unwrap()); + } + + #[test] + fn reports_an_all_lowercase_address_as_unchecksummed() { + // Not a failure of validity — it simply carries no checksum data. + let lower = "0x52908400098527886e0f7030069857d2e4169ee7"; + assert!(crate::openhuman::web3::wallet::primitives::address::evm::validate(lower).is_ok()); + assert!(!is_checksum_valid(lower).unwrap()); + } + + #[test] + fn checksumming_is_idempotent() { + for vector in EIP55_VECTORS { + let once = to_checksummed(vector).unwrap(); + assert_eq!(to_checksummed(&once).unwrap(), once); + } + } + + #[test] + fn checksumming_accepts_an_unprefixed_address_and_adds_the_prefix() { + let bare = "52908400098527886e0f7030069857d2e4169ee7"; + assert_eq!( + to_checksummed(bare).unwrap(), + "0x52908400098527886E0F7030069857D2E4169EE7" + ); + } + + #[test] + fn checksum_helpers_reject_a_malformed_address() { + assert!(to_checksummed("0xdeadbeef").is_err()); + assert!(is_checksum_valid("0xdeadbeef").is_err()); + } +} diff --git a/src/openhuman/web3/wallet/primitives/address/mod.rs b/src/openhuman/web3/wallet/primitives/address/mod.rs new file mode 100644 index 0000000000..e4798085a7 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/mod.rs @@ -0,0 +1,92 @@ +//! Per-chain address validation. +//! +//! Each submodule owns one chain's address format and exposes the same core +//! shape: a `validate` returning the trimmed address, plus whatever +//! chain-specific conversions are genuinely useful (`solana::decode`, +//! `tron::to_hex`, `btc::validate_sender`). +//! +//! [`validate`] dispatches across all of them for chain-generic callers. +//! +//! ## What validation does and does not prove +//! +//! Every function here answers one question: *is this string a well-formed +//! address on this chain*. None of them touch the network, so none can tell +//! you an account exists, is funded, or is controlled by anyone in particular. +//! +//! How much a successful validation is worth also varies sharply by chain, and +//! it is worth being explicit about because it is easy to assume otherwise: +//! +//! | Chain | Checksum | A single typo is… | +//! | --- | --- | --- | +//! | Bitcoin | yes (base58check / bech32) | caught | +//! | Tron | yes (base58check) | caught | +//! | EVM | optional (EIP-55, only if mixed-case) | usually *not* caught | +//! | Solana | none | *not reliably* caught | +//! +//! For EVM, `evm::is_checksum_valid` recovers the typo protection when the +//! caller has a mixed-case address. For Solana there is nothing to recover: +//! confirm the address out of band. + +use crate::openhuman::web3::wallet::primitives::chain::Chain; +use crate::openhuman::web3::wallet::primitives::Result; + +#[cfg(feature = "web3")] +pub mod btc; +#[cfg(feature = "web3")] +pub mod evm; +#[cfg(feature = "web3")] +pub mod solana; +#[cfg(feature = "web3")] +pub mod tron; + +/// Validate `address` for `chain`, returning it trimmed. +/// +/// Dispatches to the chain's own module. For Bitcoin this is the +/// **recipient** rule — any well-formed mainnet address; call +/// `btc::validate_sender` directly when validating a sender, since the +/// distinction has no equivalent on the other chains and cannot be expressed +/// through this entry point. +/// +/// # Errors +/// +/// Whatever the chain's own `validate` returns, plus +/// [`crate::openhuman::web3::wallet::primitives::Error::ChainNotCompiled`] if the shared `web3` feature gate is +/// disabled in this build. That case is a build fact rather than a property of +/// the address: the validation code was not compiled, so there is no answer to +/// give, and silently accepting or rejecting would be a wrong answer dressed +/// up as a real one. +/// +/// # Examples +/// +/// ``` +/// # #[cfg(feature = "web3")] { +/// use crate::openhuman::web3::wallet::primitives::{address, chain::Chain}; +/// +/// let addr = address::validate(Chain::Solana, "11111111111111111111111111111111")?; +/// assert_eq!(addr, "11111111111111111111111111111111"); +/// # } +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::Error>(()) +/// ``` +// With every chain gate off, only the `ChainNotCompiled` arm survives and +// `address` goes unread. That build is legal (a host may depend on this crate +// purely for `Chain`), so the unused binding is expected rather than a bug. +#[cfg_attr(not(feature = "web3"), allow(unused_variables))] +pub fn validate(chain: Chain, address: &str) -> Result { + match chain { + #[cfg(feature = "web3")] + Chain::Btc => btc::validate(address), + #[cfg(feature = "web3")] + Chain::Evm => evm::validate(address), + #[cfg(feature = "web3")] + Chain::Solana => solana::validate(address), + #[cfg(feature = "web3")] + Chain::Tron => tron::validate(address), + #[cfg(not(all(feature = "web3", feature = "web3", feature = "web3", feature = "web3")))] + other => Err( + crate::openhuman::web3::wallet::primitives::Error::ChainNotCompiled { chain: other }, + ), + } +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/address/solana.rs b/src/openhuman/web3/wallet/primitives/address/solana.rs new file mode 100644 index 0000000000..713e6d2da8 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/solana.rs @@ -0,0 +1,104 @@ +//! Solana address validation. +//! +//! A Solana address is an ed25519 public key — 32 raw bytes — rendered in +//! base58. There is no checksum and no version byte, so validation is exactly +//! two questions: does it decode as base58, and is the result 32 bytes. +//! +//! That absence of a checksum is worth knowing: unlike Bitcoin or Tron, a +//! single mistyped character in a Solana address usually produces *another +//! syntactically valid address*. Validation here catches malformed input, not +//! typos, and no amount of parsing can change that. + +use crate::openhuman::web3::wallet::primitives::chain::Chain; +use crate::openhuman::web3::wallet::primitives::{Error, Result}; + +/// Length in bytes of a decoded Solana address (an ed25519 public key). +pub const ADDRESS_BYTES: usize = 32; + +/// Validate a Solana address and return it trimmed. +/// +/// # Errors +/// +/// - [`Error::EmptyAddress`] if `address` is empty or all whitespace. +/// - [`Error::InvalidAddress`] if it is not base58, or does not decode to +/// exactly [`ADDRESS_BYTES`] bytes. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::solana; +/// +/// // The system program id — 32 zero bytes. +/// assert!(solana::validate("11111111111111111111111111111111").is_ok()); +/// +/// // `0` is not in the base58 alphabet. +/// assert!(solana::validate("0OIl").is_err()); +/// ``` +pub fn validate(address: &str) -> Result { + decode(address).map(|_| address.trim().to_string()) +} + +/// Validate a Solana address and return its decoded 32 bytes. +/// +/// The same check as [`validate`], for a caller that needs the key material +/// rather than the string — deriving an associated token account, say. Offered +/// so callers do not have to decode a second time immediately after +/// validating. +/// +/// # Errors +/// +/// Identical to [`validate`]. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::solana; +/// +/// let bytes = solana::decode("11111111111111111111111111111111")?; +/// assert_eq!(bytes, [0u8; 32]); +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::Error>(()) +/// ``` +pub fn decode(address: &str) -> Result<[u8; ADDRESS_BYTES]> { + let trimmed = address.trim(); + if trimmed.is_empty() { + return Err(Error::EmptyAddress { + chain: Chain::Solana, + }); + } + + let decoded = bs58::decode(trimmed) + .into_vec() + .map_err(|e| Error::InvalidAddress { + chain: Chain::Solana, + address: trimmed.to_string(), + reason: format!("not valid base58: {e}"), + })?; + + decoded + .try_into() + .map_err(|v: Vec| Error::InvalidAddress { + chain: Chain::Solana, + address: trimmed.to_string(), + reason: format!("expected {ADDRESS_BYTES} bytes, got {}", v.len()), + }) +} + +/// Render 32 raw bytes as a base58 Solana address. +/// +/// The inverse of [`decode`]. Infallible: every 32-byte array is a +/// syntactically valid address. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::solana; +/// +/// assert_eq!(solana::encode(&[0u8; 32]), "11111111111111111111111111111111"); +/// ``` +#[must_use] +pub fn encode(bytes: &[u8; ADDRESS_BYTES]) -> String { + bs58::encode(bytes).into_string() +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/address/solana/test.rs b/src/openhuman/web3/wallet/primitives/address/solana/test.rs new file mode 100644 index 0000000000..f182b52096 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/solana/test.rs @@ -0,0 +1,117 @@ +//! Unit tests for Solana address validation. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{decode, encode, validate, ADDRESS_BYTES}; +use crate::openhuman::web3::wallet::primitives::{Chain, Error}; + +/// The system program id: 32 zero bytes. +const SYSTEM_PROGRAM: &str = "11111111111111111111111111111111"; +/// The SPL token program id — a real 32-byte key with a full alphabet. +const TOKEN_PROGRAM: &str = "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"; + +#[test] +fn accepts_real_addresses() { + for addr in [SYSTEM_PROGRAM, TOKEN_PROGRAM] { + assert_eq!(validate(addr).unwrap(), addr); + } +} + +#[test] +fn trims_surrounding_whitespace() { + assert_eq!( + validate(&format!(" {TOKEN_PROGRAM}\n")).unwrap(), + TOKEN_PROGRAM + ); +} + +#[test] +fn rejects_an_empty_address() { + assert_eq!( + validate(" ").unwrap_err(), + Error::EmptyAddress { + chain: Chain::Solana + } + ); +} + +#[test] +fn rejects_characters_outside_the_base58_alphabet() { + // `0`, `O`, `I` and `l` are excluded from base58 precisely because they + // are visually ambiguous. + for bad in ["0OIl", "hello world", "not!base58"] { + match validate(bad).unwrap_err() { + Error::InvalidAddress { chain, reason, .. } => { + assert_eq!(chain, Chain::Solana); + assert!(reason.contains("base58"), "reason was {reason:?}"); + } + other => panic!("expected InvalidAddress for {bad:?}, got {other:?}"), + } + } +} + +#[test] +fn rejects_a_decoded_length_other_than_32_bytes() { + // Valid base58, wrong length — the check a base58 decode alone misses. + let short = encode_arbitrary(&[1u8; 16]); + match validate(&short).unwrap_err() { + Error::InvalidAddress { reason, .. } => { + assert!(reason.contains("32"), "reason was {reason:?}"); + assert!( + reason.contains("16"), + "reason should report the actual length" + ); + } + other => panic!("expected InvalidAddress, got {other:?}"), + } + + let long = encode_arbitrary(&[1u8; 33]); + assert!(validate(&long).is_err()); +} + +#[test] +fn decode_returns_the_raw_key_bytes() { + assert_eq!(decode(SYSTEM_PROGRAM).unwrap(), [0u8; ADDRESS_BYTES]); +} + +#[test] +fn encode_and_decode_round_trip() { + let mut bytes = [0u8; ADDRESS_BYTES]; + for (i, b) in bytes.iter_mut().enumerate() { + *b = u8::try_from(i).unwrap(); + } + assert_eq!(decode(&encode(&bytes)).unwrap(), bytes); +} + +#[test] +fn decode_rejects_exactly_what_validate_rejects() { + // The two share a code path; this pins that they cannot drift apart. + for input in ["", " ", "0OIl", &encode_arbitrary(&[7u8; 31])] { + assert_eq!( + validate(input).is_err(), + decode(input).is_err(), + "validate and decode disagreed on {input:?}" + ); + } +} + +#[test] +fn a_single_character_typo_is_not_caught() { + // Documenting a real property of the chain, not endorsing it: Solana + // addresses carry no checksum, so a typo usually yields another valid + // address. Callers must confirm addresses out of band. + let mut chars: Vec = TOKEN_PROGRAM.chars().collect(); + chars[4] = if chars[4] == 'a' { 'b' } else { 'a' }; + let typo: String = chars.into_iter().collect(); + assert_ne!(typo, TOKEN_PROGRAM); + assert!( + validate(&typo).is_ok(), + "a Solana typo is indistinguishable from a real address" + ); +} + +/// Base58-encode arbitrary bytes, bypassing the 32-byte contract, so tests can +/// build wrong-length-but-valid-base58 inputs. +fn encode_arbitrary(bytes: &[u8]) -> String { + bs58::encode(bytes).into_string() +} diff --git a/src/openhuman/web3/wallet/primitives/address/test.rs b/src/openhuman/web3/wallet/primitives/address/test.rs new file mode 100644 index 0000000000..15516d6afc --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/test.rs @@ -0,0 +1,96 @@ +//! Unit tests for chain-generic address dispatch. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::validate; +use crate::openhuman::web3::wallet::primitives::{Chain, Error}; + +/// One valid mainnet address per chain. +const FIXTURES: [(Chain, &str); 4] = [ + (Chain::Btc, "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4"), + (Chain::Evm, "0x52908400098527886E0F7030069857D2E4169EE7"), + (Chain::Solana, "11111111111111111111111111111111"), + (Chain::Tron, "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t"), +]; + +/// Whether the feature gate behind `chain` is on in this build. +/// +/// The dispatch assertions have to reflect the gated contract: a chain whose +/// gate is off is *supposed* to answer `ChainNotCompiled`, so the tests expect +/// success exactly for the chains that are compiled in. +const fn chain_enabled(chain: Chain) -> bool { + match chain { + #[cfg(feature = "web3")] + Chain::Btc => true, + #[cfg(feature = "web3")] + Chain::Evm => true, + #[cfg(feature = "web3")] + Chain::Solana => true, + #[cfg(feature = "web3")] + Chain::Tron => true, + #[cfg(not(all(feature = "web3", feature = "web3", feature = "web3", feature = "web3")))] + _ => false, + } +} + +#[test] +fn dispatches_every_chain_to_its_own_validator() { + for (chain, address) in FIXTURES { + if chain_enabled(chain) { + assert_eq!( + validate(chain, address).unwrap(), + address, + "{chain} dispatch failed" + ); + } else { + assert!( + matches!( + validate(chain, address), + Err(Error::ChainNotCompiled { .. }) + ), + "{chain} gate is off in this build, so validation must report \ + ChainNotCompiled, not validate" + ); + } + } +} + +#[test] +fn every_known_chain_has_a_fixture() { + // If `Chain::ALL` grows, this test fails until the new chain is covered + // above — otherwise a new variant would silently go untested. + assert_eq!(Chain::ALL.len(), FIXTURES.len()); + for chain in Chain::ALL { + assert!( + FIXTURES.iter().any(|(c, _)| c == chain), + "no dispatch fixture for {chain}" + ); + } +} + +#[test] +fn an_address_from_the_wrong_chain_is_rejected() { + // The dispatch must actually route: a Solana address handed to the Tron + // arm has to fail, or the match is not doing its job. + for (chain, address) in FIXTURES { + for (other_chain, _) in FIXTURES { + if chain == other_chain { + continue; + } + assert!( + validate(other_chain, address).is_err(), + "{chain} address {address} was wrongly accepted as {other_chain}" + ); + } + } +} + +#[test] +fn dispatch_rejects_empty_input_on_every_chain() { + for (chain, _) in FIXTURES { + assert!( + validate(chain, " ").is_err(), + "{chain} accepted whitespace" + ); + } +} diff --git a/src/openhuman/web3/wallet/primitives/address/tron.rs b/src/openhuman/web3/wallet/primitives/address/tron.rs new file mode 100644 index 0000000000..0f0c500e24 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/tron.rs @@ -0,0 +1,147 @@ +//! Tron address validation and hex conversion. +//! +//! Tron addresses come in two forms, and any code touching the chain deals +//! with both: +//! +//! - **Base58check** (`T…`) — the user-facing form. 21 bytes: a `0x41` version +//! prefix plus a 20-byte payload, with a 4-byte checksum appended. +//! - **Hex** (`41…`) — the same 21 bytes, hex-encoded. This is what the +//! `TronGrid` API speaks. +//! +//! [`to_hex`] converts between them. Unlike Solana, Tron addresses *are* +//! checksummed, so a mistyped address is reliably caught here rather than +//! silently naming a different account. + +use crate::openhuman::web3::wallet::primitives::chain::Chain; +use crate::openhuman::web3::wallet::primitives::{Error, Result}; + +/// Tron mainnet address version prefix. +/// +/// Every decoded mainnet address starts with this byte; base58check decoding +/// verifies it, which is what makes a testnet or foreign-chain address fail +/// rather than decode to something plausible. +pub const MAINNET_PREFIX: u8 = 0x41; + +/// Length in bytes of a decoded Tron address: the version prefix plus a +/// 20-byte payload. +pub const ADDRESS_BYTES: usize = 21; + +/// Validate a Tron mainnet address and return it trimmed. +/// +/// # Errors +/// +/// - [`Error::EmptyAddress`] if `address` is empty or all whitespace. +/// - [`Error::InvalidAddress`] if base58check decoding fails — a bad checksum, +/// a non-base58 character, or a version byte other than +/// [`MAINNET_PREFIX`] — or if the payload is not [`ADDRESS_BYTES`] bytes. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::tron; +/// +/// assert!(tron::validate("TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t").is_ok()); +/// +/// // One character changed: the checksum catches it. +/// assert!(tron::validate("TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6u").is_err()); +/// ``` +pub fn validate(address: &str) -> Result { + decode(address).map(|_| address.trim().to_string()) +} + +/// Validate a Tron address and return its decoded 21 bytes, version prefix +/// included. +/// +/// # Errors +/// +/// Identical to [`validate`]. +pub fn decode(address: &str) -> Result<[u8; ADDRESS_BYTES]> { + let trimmed = address.trim(); + if trimmed.is_empty() { + return Err(Error::EmptyAddress { chain: Chain::Tron }); + } + + let decoded = bs58::decode(trimmed) + .with_check(Some(MAINNET_PREFIX)) + .into_vec() + .map_err(|e| Error::InvalidAddress { + chain: Chain::Tron, + address: trimmed.to_string(), + reason: format!("base58check decoding failed: {e}"), + })?; + + decoded + .try_into() + .map_err(|v: Vec| Error::InvalidAddress { + chain: Chain::Tron, + address: trimmed.to_string(), + reason: format!( + "expected {ADDRESS_BYTES} bytes after base58check, got {}", + v.len() + ), + }) +} + +/// Convert a base58check Tron address to its hex form. +/// +/// The result is 42 lowercase hex digits — the 21 decoded bytes including the +/// `41` version prefix, with no `0x`. That is the form the `TronGrid` API +/// expects; it is **not** an EVM address, despite the superficial resemblance. +/// +/// # Errors +/// +/// Identical to [`validate`]. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::tron; +/// +/// let hex = tron::to_hex("TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t")?; +/// assert_eq!(hex.len(), 42); +/// assert!(hex.starts_with("41"), "the version prefix is retained"); +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::Error>(()) +/// ``` +pub fn to_hex(address: &str) -> Result { + Ok(hex::encode(decode(address)?)) +} + +/// Render 21 decoded bytes as a base58check Tron address. +/// +/// The inverse of [`decode`]. The input must be a full mainnet address — +/// version prefix included — so its first byte must be [`MAINNET_PREFIX`]. +/// Enforcing that here means every successful result round-trips through both +/// [`decode`] and [`validate`]. +/// +/// # Errors +/// +/// - [`Error::WrongNetwork`] if the first byte is not [`MAINNET_PREFIX`]: the +/// bytes are then a well-formed address for some other Tron network, not +/// mainnet. +/// +/// # Examples +/// +/// ``` +/// use crate::openhuman::web3::wallet::primitives::address::tron; +/// +/// let bytes = tron::decode("TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t")?; +/// assert_eq!(tron::encode(&bytes)?, "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t"); +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::Error>(()) +/// ``` +pub fn encode(bytes: &[u8; ADDRESS_BYTES]) -> Result { + if bytes[0] != MAINNET_PREFIX { + return Err(Error::WrongNetwork { + chain: Chain::Tron, + address: hex::encode(bytes), + expected: "mainnet".to_string(), + reason: format!( + "version prefix is {:#04x}, expected {MAINNET_PREFIX:#04x}", + bytes[0] + ), + }); + } + Ok(bs58::encode(bytes).with_check().into_string()) +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/address/tron/test.rs b/src/openhuman/web3/wallet/primitives/address/tron/test.rs new file mode 100644 index 0000000000..91c9f3b812 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/address/tron/test.rs @@ -0,0 +1,145 @@ +//! Unit tests for Tron address validation and hex conversion. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{decode, encode, to_hex, validate, ADDRESS_BYTES, MAINNET_PREFIX}; +use crate::openhuman::web3::wallet::primitives::{Chain, Error}; + +/// The USDT TRC20 contract address — a real, checksummed mainnet address. +const USDT: &str = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t"; + +#[test] +fn accepts_a_real_mainnet_address() { + assert_eq!(validate(USDT).unwrap(), USDT); +} + +#[test] +fn trims_surrounding_whitespace() { + assert_eq!(validate(&format!(" {USDT}\n")).unwrap(), USDT); +} + +#[test] +fn rejects_an_empty_address() { + assert_eq!( + validate(" ").unwrap_err(), + Error::EmptyAddress { chain: Chain::Tron } + ); +} + +#[test] +fn rejects_a_mistyped_address_via_its_checksum() { + // Unlike Solana, Tron addresses are checksummed, so a typo is caught. + let mut chars: Vec = USDT.chars().collect(); + let last = chars.len() - 1; + chars[last] = if chars[last] == 'u' { 'v' } else { 'u' }; + let typo: String = chars.into_iter().collect(); + assert_ne!(typo, USDT, "the fixture must actually differ"); + + match validate(&typo).unwrap_err() { + Error::InvalidAddress { chain, address, .. } => { + assert_eq!(chain, Chain::Tron); + assert_eq!(address, typo); + } + other => panic!("expected InvalidAddress, got {other:?}"), + } +} + +#[test] +fn rejects_a_non_base58_address() { + assert!(matches!( + validate("not!an!address").unwrap_err(), + Error::InvalidAddress { .. } + )); +} + +#[test] +fn rejects_an_address_with_a_foreign_version_prefix() { + // Same 20-byte payload, a different version byte. Base58check verifies the + // prefix, which is what stops a foreign-chain address decoding to + // something plausible. + let mut bytes = [0u8; ADDRESS_BYTES]; + bytes[0] = 0x30; + let foreign = bs58::encode(bytes).with_check().into_string(); + assert!( + validate(&foreign).is_err(), + "a non-{MAINNET_PREFIX:#x} prefix must be rejected" + ); +} + +#[test] +fn rejects_a_base58check_value_with_the_right_prefix_but_wrong_length() { + let short = bs58::encode([MAINNET_PREFIX]).with_check().into_string(); + assert!(matches!( + decode(&short), + Err(Error::InvalidAddress { + chain: Chain::Tron, + .. + }) + )); +} + +#[test] +fn decode_retains_the_version_prefix() { + let bytes = decode(USDT).unwrap(); + assert_eq!(bytes.len(), ADDRESS_BYTES); + assert_eq!(bytes[0], MAINNET_PREFIX); +} + +#[test] +fn encode_and_decode_round_trip() { + assert_eq!(encode(&decode(USDT).unwrap()).unwrap(), USDT); +} + +#[test] +fn encode_rejects_a_non_mainnet_version_prefix() { + // `encode` must not mint an address that `validate` would reject: with any + // first byte other than the mainnet prefix the result is well-formed + // base58check for some *other* Tron network. + let mut bytes = [0u8; ADDRESS_BYTES]; + bytes[0] = 0x30; + match encode(&bytes).unwrap_err() { + Error::WrongNetwork { + chain, + address, + expected, + reason, + } => { + assert_eq!(chain, Chain::Tron); + assert!(address.starts_with("30"), "hex form: {address}"); + assert_eq!(expected, "mainnet"); + assert!(reason.contains(&format!("{MAINNET_PREFIX:#04x}"))); + } + other => panic!("expected WrongNetwork, got {other:?}"), + } +} + +#[test] +fn to_hex_produces_the_trongrid_form() { + let hex = to_hex(USDT).unwrap(); + // 21 bytes, two hex digits each — and no `0x`, because this is not an EVM + // address despite the resemblance. + assert_eq!(hex.len(), ADDRESS_BYTES * 2); + assert!(!hex.starts_with("0x")); + assert!( + hex.starts_with("41"), + "the version prefix is retained: {hex}" + ); + assert!(hex.chars().all(|c| c.is_ascii_hexdigit())); + assert_eq!(hex, hex.to_lowercase(), "hex output is lowercase"); +} + +#[test] +fn to_hex_rejects_exactly_what_validate_rejects() { + for input in [ + "", + " ", + "not!base58", + "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6u", + ] { + assert_eq!( + validate(input).is_err(), + to_hex(input).is_err(), + "validate and to_hex disagreed on {input:?}" + ); + } +} diff --git a/src/openhuman/web3/wallet/primitives/chain/mod.rs b/src/openhuman/web3/wallet/primitives/chain/mod.rs new file mode 100644 index 0000000000..6a3c7f9383 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/chain/mod.rs @@ -0,0 +1,102 @@ +//! The set of chains this crate understands. +//! +//! [`Chain`] exists so errors can name the chain they came from without every +//! variant carrying a stringly-typed label, and so a host can drive +//! chain-generic code — a dispatch table, a UI picker — off one enum rather +//! than its own parallel copy. +//! +//! It is deliberately **not** feature-gated. A host compiled with only the +//! `solana` gate should still be able to name and match on `Chain::Btc` +//! (in a config file it round-trips, say) without that failing to compile; +//! only the *validation functions* disappear with their gates. + +use std::fmt; +use std::str::FromStr; + +/// A blockchain this crate has address support for. +/// +/// Serde support is conditional so the enum stays dependency-free in builds +/// that do not need it. The representation is the lowercase variant name +/// (`"btc"`, `"evm"`, …), matching [`FromStr`] and [`fmt::Display`] below, so a +/// value written by one and read by the other agrees — this type crosses a +/// host/backend boundary in [`crate::openhuman::web3::wallet::primitives::wire`], where a mismatch between the text +/// and JSON forms would be a runtime deserialization failure. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] +#[serde(rename_all = "lowercase")] +#[non_exhaustive] +pub enum Chain { + /// Bitcoin (mainnet). + Btc, + /// An EVM chain — Ethereum and every address-compatible network. + /// + /// One variant covers all of them because the address format is identical + /// across EVM chains; nothing about validating an address distinguishes + /// Ethereum from Polygon or Base. + Evm, + /// Solana (mainnet-beta). + Solana, + /// Tron (mainnet). + Tron, +} + +impl Chain { + /// Every chain this crate knows, in declaration order. + /// + /// Useful for a host enumerating supported chains. This is the full set + /// regardless of which feature gates are enabled — see the module docs. + pub const ALL: &'static [Self] = &[Self::Btc, Self::Evm, Self::Solana, Self::Tron]; + + /// The chain's lowercase machine-readable name (`"btc"`, `"evm"`, + /// `"solana"`, `"tron"`). + /// + /// This is the form [`Chain::from_str`] parses, so `chain.as_str()` always + /// round-trips. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Btc => "btc", + Self::Evm => "evm", + Self::Solana => "solana", + Self::Tron => "tron", + } + } +} + +impl fmt::Display for Chain { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Returned by [`Chain::from_str`] when the input names no known chain. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unknown chain '{0}'")] +pub struct UnknownChain(pub String); + +impl FromStr for Chain { + type Err = UnknownChain; + + /// Parse a chain from its machine-readable name, case-insensitively. + /// + /// `"ethereum"` and `"eth"` are accepted as aliases for [`Chain::Evm`], + /// and `"bitcoin"` for [`Chain::Btc`], because those are the spellings + /// that show up in user-facing config. + /// + /// # Errors + /// + /// Returns [`UnknownChain`] if `s` names no known chain. + fn from_str(s: &str) -> std::result::Result { + match s.trim().to_ascii_lowercase().as_str() { + "btc" | "bitcoin" => Ok(Self::Btc), + "evm" | "eth" | "ethereum" => Ok(Self::Evm), + "solana" | "sol" => Ok(Self::Solana), + "tron" | "trx" => Ok(Self::Tron), + other => Err(UnknownChain(other.to_string())), + } + } +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/chain/test.rs b/src/openhuman/web3/wallet/primitives/chain/test.rs new file mode 100644 index 0000000000..7267b7103b --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/chain/test.rs @@ -0,0 +1,57 @@ +//! Unit tests for the [`Chain`](super::Chain) enum. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::str::FromStr; + +use super::{Chain, UnknownChain}; + +#[test] +fn as_str_round_trips_through_from_str() { + for chain in Chain::ALL { + assert_eq!(Chain::from_str(chain.as_str()).unwrap(), *chain); + } +} + +#[test] +fn display_matches_as_str() { + for chain in Chain::ALL { + assert_eq!(chain.to_string(), chain.as_str()); + } +} + +#[test] +fn all_contains_no_duplicates() { + let mut seen = Chain::ALL.to_vec(); + seen.sort_unstable(); + seen.dedup(); + assert_eq!(seen.len(), Chain::ALL.len()); +} + +#[test] +fn parsing_is_case_and_whitespace_insensitive() { + assert_eq!(Chain::from_str(" BTC \n").unwrap(), Chain::Btc); + assert_eq!(Chain::from_str("SoLaNa").unwrap(), Chain::Solana); +} + +#[test] +fn common_aliases_parse() { + // These are the spellings that turn up in user-facing config. + for (input, expected) in [ + ("bitcoin", Chain::Btc), + ("eth", Chain::Evm), + ("ethereum", Chain::Evm), + ("sol", Chain::Solana), + ("trx", Chain::Tron), + ] { + assert_eq!(Chain::from_str(input).unwrap(), expected, "alias {input}"); + } +} + +#[test] +fn an_unknown_name_is_reported_with_the_input() { + assert_eq!( + Chain::from_str("dogecoin").unwrap_err(), + UnknownChain("dogecoin".to_string()) + ); +} diff --git a/src/openhuman/web3/wallet/primitives/eip712/mod.rs b/src/openhuman/web3/wallet/primitives/eip712/mod.rs new file mode 100644 index 0000000000..a99009d8c0 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/eip712/mod.rs @@ -0,0 +1,189 @@ +//! EIP-712 typed-data hashing, and the EIP-3009 payload x402 signs. +//! +//! # Why this is here rather than in a chain library +//! +//! EIP-712 is a hashing scheme, not a chain client. Everything below is +//! keccak-256 over a fixed byte layout — there is no RPC, no signing, and no +//! elliptic curve involved. Hosting it here means the x402 payment path needs +//! `sha3` and nothing else, where routing it through a full Ethereum library +//! costs an ABI encoder, a bignum type, a signer stack, and their tails. +//! +//! # Integers are big-endian `[u8; 32]`, deliberately +//! +//! EIP-712 encodes every `uint256` as a 32-byte big-endian word, so that is the +//! type this module takes. Introducing a bignum just to convert it back to the +//! same 32 bytes would add a dependency to this crate and force one on every +//! caller. [`u256_from_u64`] and [`u256_from_decimal`] cover the two ways a +//! caller actually has the value. +//! +//! # Nothing here signs +//! +//! [`signing_digest`] returns the 32 bytes to sign and stops. That is the same +//! split the rest of this crate makes — see [`crate::openhuman::web3::wallet::primitives::wire`] — and it is what +//! lets the payload be built somewhere the signing key is not. + +use sha3::{Digest, Keccak256}; + +/// `keccak256("EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)")`. +/// +/// Pinned rather than computed at each call: it is a published constant, and a +/// test below recomputes it, so a typo in the type string is caught here rather +/// than as a signature a contract silently rejects. +const DOMAIN_TYPE_HASH: [u8; 32] = [ + 0x8b, 0x73, 0xc3, 0xc6, 0x9b, 0xb8, 0xfe, 0x3d, 0x51, 0x2e, 0xcc, 0x4c, 0xf7, 0x59, 0xcc, 0x79, + 0x23, 0x9f, 0x7b, 0x17, 0x9b, 0x0f, 0xfa, 0xca, 0xa9, 0xa7, 0x5d, 0x52, 0x2b, 0x39, 0x40, 0x0f, +]; + +/// The EIP-712 type string for the EIP-3009 authorization x402 uses. +const TRANSFER_WITH_AUTHORIZATION_TYPE: &[u8] = b"TransferWithAuthorization(address from,address to,uint256 value,uint256 validAfter,uint256 validBefore,bytes32 nonce)"; + +/// The EIP-712 domain string, kept beside its pinned hash. +/// +/// Test-only, and that is the point: production code uses [`DOMAIN_TYPE_HASH`] +/// directly rather than hashing this on every call, and the test re-derives the +/// hash from this string to prove the two agree. Keeping the string here is +/// what makes pinning the hash safe instead of merely fast — a typo in either +/// one fails the test rather than silently changing every signature. +#[cfg(test)] +const DOMAIN_TYPE: &[u8] = + b"EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)"; + +/// A 32-byte big-endian unsigned integer, as EIP-712 encodes `uint256`. +pub type U256Bytes = [u8; 32]; + +/// An EVM address as its raw 20 bytes. +pub type Address20 = [u8; 20]; + +/// Widen a `u64` into the 32-byte big-endian form EIP-712 wants. +#[must_use] +pub fn u256_from_u64(value: u64) -> U256Bytes { + let mut out = [0u8; 32]; + out[24..].copy_from_slice(&value.to_be_bytes()); + out +} + +/// Parse a base-10 integer string into the 32-byte big-endian form. +/// +/// Token amounts arrive as decimal strings — a `u64` cannot hold 18-decimal +/// values — so this does the widening without a bignum dependency, by long +/// multiplication over the 32 bytes. +/// +/// # Errors +/// +/// [`Error::InvalidAmount`] if `value` is empty, holds a non-digit, or does not +/// fit in 256 bits. +pub fn u256_from_decimal(value: &str) -> Result { + let trimmed = value.trim(); + if trimmed.is_empty() || !trimmed.bytes().all(|b| b.is_ascii_digit()) { + return Err(Error::InvalidAmount { + reason: "expected a base-10 integer".to_string(), + }); + } + + let mut out = [0u8; 32]; + for digit in trimmed.bytes().map(|b| u32::from(b - b'0')) { + // out = out * 10 + digit, big-endian, carrying from the least + // significant byte upwards. + let mut carry = digit; + for byte in out.iter_mut().rev() { + let product = u32::from(*byte) * 10 + carry; + *byte = u8::try_from(product & 0xff).unwrap_or(0); + carry = product >> 8; + } + if carry != 0 { + return Err(Error::InvalidAmount { + reason: "value does not fit in 256 bits".to_string(), + }); + } + } + Ok(out) +} + +/// Why an EIP-712 payload could not be built. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// An amount was not a base-10 integer, or overflowed 256 bits. + #[error("invalid amount: {reason}")] + InvalidAmount { + /// What was wrong with it. + reason: String, + }, +} + +/// Result alias for this module. +pub type Result = std::result::Result; + +/// The EIP-712 domain separator. +/// +/// `name` and `version` are the token contract's, not the caller's choice: USDC +/// uses `("USD Coin", "2")`, but an x402 `extra` may name different ones, and a +/// mismatch produces a signature the contract rejects rather than an error +/// anything local can detect. +#[must_use] +pub fn domain_separator( + verifying_contract: Address20, + chain_id: u64, + name: &str, + version: &str, +) -> [u8; 32] { + let mut encoded = Vec::with_capacity(5 * 32); + encoded.extend_from_slice(&DOMAIN_TYPE_HASH); + encoded.extend_from_slice(&keccak(name.as_bytes())); + encoded.extend_from_slice(&keccak(version.as_bytes())); + encoded.extend_from_slice(&u256_from_u64(chain_id)); + encoded.extend_from_slice(&left_pad_address(verifying_contract)); + keccak(&encoded) +} + +/// The EIP-3009 `TransferWithAuthorization` struct hash. +#[must_use] +pub fn transfer_with_authorization_hash( + from: Address20, + to: Address20, + value: U256Bytes, + valid_after: U256Bytes, + valid_before: U256Bytes, + nonce: [u8; 32], +) -> [u8; 32] { + let mut encoded = Vec::with_capacity(7 * 32); + encoded.extend_from_slice(&keccak(TRANSFER_WITH_AUTHORIZATION_TYPE)); + encoded.extend_from_slice(&left_pad_address(from)); + encoded.extend_from_slice(&left_pad_address(to)); + encoded.extend_from_slice(&value); + encoded.extend_from_slice(&valid_after); + encoded.extend_from_slice(&valid_before); + encoded.extend_from_slice(&nonce); + keccak(&encoded) +} + +/// The 32 bytes a caller signs: `keccak256(0x19 0x01 ‖ domain ‖ struct)`. +/// +/// The `0x1901` prefix is what keeps a typed-data signature from ever being +/// replayable as a transaction signature — it makes the preimage impossible to +/// confuse with an RLP-encoded transaction. +/// +/// Already hashed: sign it with a "prehash" entry point, never by hashing again. +#[must_use] +pub fn signing_digest(domain_separator: [u8; 32], struct_hash: [u8; 32]) -> [u8; 32] { + let mut preimage = Vec::with_capacity(2 + 64); + preimage.extend_from_slice(&[0x19, 0x01]); + preimage.extend_from_slice(&domain_separator); + preimage.extend_from_slice(&struct_hash); + keccak(&preimage) +} + +/// Keccak-256. +fn keccak(bytes: &[u8]) -> [u8; 32] { + Keccak256::digest(bytes).into() +} + +/// An address as a left-padded 32-byte word, which is how EIP-712 encodes it. +fn left_pad_address(address: Address20) -> [u8; 32] { + let mut out = [0u8; 32]; + out[12..].copy_from_slice(&address); + out +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/eip712/test.rs b/src/openhuman/web3/wallet/primitives/eip712/test.rs new file mode 100644 index 0000000000..9dcc92c118 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/eip712/test.rs @@ -0,0 +1,207 @@ +//! Tests for EIP-712 hashing. +//! +//! A wrong hash here is not a crash — it is a well-formed signature over +//! something other than the intended payment, which the contract rejects with +//! no explanation, or worse, accepts. So the constants are checked against the +//! specifications rather than against this module's own output. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{ + domain_separator, keccak, signing_digest, transfer_with_authorization_hash, u256_from_decimal, + u256_from_u64, Error, DOMAIN_TYPE, DOMAIN_TYPE_HASH, TRANSFER_WITH_AUTHORIZATION_TYPE, +}; + +fn hex(bytes: &[u8]) -> String { + bytes.iter().fold(String::new(), |mut out, b| { + use std::fmt::Write as _; + let _ = write!(out, "{b:02x}"); + out + }) +} + +#[test] +fn the_pinned_domain_type_hash_matches_its_type_string() { + // The constant is pinned so a typo in the type string cannot silently + // change every signature this module produces. This is the test that makes + // pinning safe rather than merely convenient. + assert_eq!(keccak(DOMAIN_TYPE), DOMAIN_TYPE_HASH); +} + +#[test] +fn the_domain_type_hash_is_the_published_constant() { + assert_eq!( + hex(&DOMAIN_TYPE_HASH), + "8b73c3c69bb8fe3d512ecc4cf759cc79239f7b179b0ffacaa9a75d522b39400f" + ); +} + +#[test] +fn the_eip3009_type_hash_is_the_published_constant() { + // From EIP-3009. A wrong type hash produces a signature that every + // conforming token contract refuses. + assert_eq!( + hex(&keccak(TRANSFER_WITH_AUTHORIZATION_TYPE)), + "7c7c6cdb67a18743f49ec6fa9b35f50d52ed05cbed4cc592e13b44501c1a2267" + ); +} + +#[test] +fn a_u64_widens_into_the_low_eight_bytes() { + let widened = u256_from_u64(1); + assert_eq!(widened[31], 1); + assert!(widened[..31].iter().all(|b| *b == 0)); + + assert_eq!( + hex(&u256_from_u64(u64::MAX)), + "000000000000000000000000000000000000000000000000ffffffffffffffff" + ); +} + +#[test] +fn a_decimal_string_widens_the_same_way_a_u64_does() { + // The two paths must agree wherever they overlap, or an amount's encoding + // would depend on which one the caller happened to use. + for value in [0u64, 1, 42, 1_000_000, u64::MAX] { + assert_eq!( + u256_from_decimal(&value.to_string()).unwrap(), + u256_from_u64(value), + "decimal and u64 widening disagree for {value}" + ); + } +} + +#[test] +fn a_decimal_string_carries_beyond_sixty_four_bits() { + // The reason the decimal path exists: an 18-decimal token amount does not + // fit in a u64. + let one_ether = "1000000000000000000000000000000000000000"; + let widened = u256_from_decimal(one_ether).unwrap(); + assert!( + widened[..8].iter().any(|b| *b != 0) || widened[8..24].iter().any(|b| *b != 0), + "a value past 2^64 must occupy the high bytes: {}", + hex(&widened) + ); + + // 2^128, checked exactly. + assert_eq!( + hex(&u256_from_decimal("340282366920938463463374607431768211456").unwrap()), + "0000000000000000000000000000000100000000000000000000000000000000" + ); +} + +#[test] +fn the_largest_representable_value_is_accepted_and_the_next_is_not() { + let max = "1157920892373161954235709850086879078532699846656405640394575840079131296399\ + 35"; + let max = max.replace(char::is_whitespace, ""); + assert_eq!(hex(&u256_from_decimal(&max).unwrap()), "ff".repeat(32)); + + // 2^256 exactly: one past the top. + let overflow = "115792089237316195423570985008687907853269984665640564039457584007913129639936"; + assert!(matches!( + u256_from_decimal(overflow).unwrap_err(), + Error::InvalidAmount { .. } + )); +} + +#[test] +fn a_non_numeric_amount_is_refused_rather_than_silently_zero() { + for bad in ["", " ", "12a", "-1", "1.5", "0x10"] { + assert!( + matches!(u256_from_decimal(bad), Err(Error::InvalidAmount { .. })), + "{bad:?} should be refused" + ); + } +} + +#[test] +fn the_domain_separator_depends_on_every_one_of_its_inputs() { + // Each field is part of the replay boundary: the same authorization must + // not verify on another chain, another contract, or another token. + let contract = [0x11u8; 20]; + let base = domain_separator(contract, 1, "USD Coin", "2"); + + assert_ne!(base, domain_separator([0x22u8; 20], 1, "USD Coin", "2")); + assert_ne!(base, domain_separator(contract, 8453, "USD Coin", "2")); + assert_ne!(base, domain_separator(contract, 1, "USDC", "2")); + assert_ne!(base, domain_separator(contract, 1, "USD Coin", "1")); +} + +#[test] +fn the_struct_hash_depends_on_every_one_of_its_inputs() { + let base = transfer_with_authorization_hash( + [0x11; 20], + [0x22; 20], + u256_from_u64(100), + u256_from_u64(0), + u256_from_u64(9_999), + [0x33; 32], + ); + + // Recipient and value especially: a hash insensitive to either would let a + // payment be redirected or resized after signing. + assert_ne!( + base, + transfer_with_authorization_hash( + [0x11; 20], + [0xaa; 20], + u256_from_u64(100), + u256_from_u64(0), + u256_from_u64(9_999), + [0x33; 32], + ) + ); + assert_ne!( + base, + transfer_with_authorization_hash( + [0x11; 20], + [0x22; 20], + u256_from_u64(101), + u256_from_u64(0), + u256_from_u64(9_999), + [0x33; 32], + ) + ); + assert_ne!( + base, + transfer_with_authorization_hash( + [0x11; 20], + [0x22; 20], + u256_from_u64(100), + u256_from_u64(0), + u256_from_u64(9_999), + [0x44; 32], + ) + ); +} + +#[test] +fn the_signing_digest_is_prefixed_so_it_cannot_be_a_transaction() { + // The 0x1901 prefix is the whole reason a typed-data signature cannot be + // replayed as a transaction signature. + let domain = [0x11u8; 32]; + let structure = [0x22u8; 32]; + + let mut preimage = vec![0x19, 0x01]; + preimage.extend_from_slice(&domain); + preimage.extend_from_slice(&structure); + + assert_eq!(signing_digest(domain, structure), keccak(&preimage)); + // And it must not be a bare hash of the concatenation. + assert_ne!( + signing_digest(domain, structure), + keccak(&[domain, structure].concat()) + ); +} + +#[test] +fn swapping_the_domain_and_struct_hashes_changes_the_digest() { + // Ordering inside the preimage is load-bearing and easy to get backwards. + let domain = [0x11u8; 32]; + let structure = [0x22u8; 32]; + assert_ne!( + signing_digest(domain, structure), + signing_digest(structure, domain) + ); +} diff --git a/src/openhuman/web3/wallet/primitives/error/mod.rs b/src/openhuman/web3/wallet/primitives/error/mod.rs new file mode 100644 index 0000000000..85e3c8c7ea --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/error/mod.rs @@ -0,0 +1,101 @@ +//! Crate-wide error and result types. +//! +//! Every fallible public function in this crate returns [`Result`], and every +//! failure mode is a distinct [`Error`] variant. Add a variant rather than +//! encoding new context into an existing message: callers match on variants, +//! and message text is not a stable API. +//! +//! Errors carry the offending input verbatim. That is a deliberate choice for +//! this crate: an address is public data, and a caller diagnosing a rejected +//! address needs to see exactly what was rejected — a truncated or elided +//! address turns a one-line fix into a debugging session. **Nothing in this +//! crate ever puts a secret in an error**; key-material failures report the +//! failing step, never the material. + +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// Errors returned by this crate. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// An address was empty or contained only whitespace. + #[error("{chain} address is empty")] + EmptyAddress { + /// The chain the address was being validated for. + chain: Chain, + }, + + /// An address was not well-formed for its chain. + /// + /// Covers every syntactic rejection: a bad base58 checksum, a wrong + /// length, a non-hex character, an invalid bech32 payload. + #[error("invalid {chain} address '{address}': {reason}")] + InvalidAddress { + /// The chain the address was being validated for. + chain: Chain, + /// The rejected address, verbatim. + address: String, + /// Why it was rejected. + reason: String, + }, + + /// An address was well-formed but belongs to the wrong network — a + /// testnet or regtest address where a mainnet one is required. + /// + /// Separate from [`Error::InvalidAddress`] because it is the one failure a + /// caller is likely to *handle* rather than merely report: it means the + /// user is pointed at the wrong network, not that they typo'd. + #[error("{chain} address '{address}' is not on {expected}: {reason}")] + WrongNetwork { + /// The chain the address was being validated for. + chain: Chain, + /// The rejected address, verbatim. + address: String, + /// The network that was required. + expected: String, + /// Detail from the underlying parser. + reason: String, + }, + + /// An address is well-formed but its type is not supported for the + /// requested role. + /// + /// Raised by `address::btc::validate_sender`: signing is only + /// implemented for P2WPKH, so a P2TR or P2SH address is a perfectly valid + /// *recipient* and an unusable *sender*. + #[error("{chain} address '{address}' is not supported as a sender: {reason}")] + UnsupportedAddressType { + /// The chain the address was being validated for. + chain: Chain, + /// The rejected address, verbatim. + address: String, + /// Which address types are supported instead. + reason: String, + }, + + /// The chain's feature gate was disabled when this crate was built. + /// + /// Only [`crate::openhuman::web3::wallet::primitives::address::validate`] can return this, and only for a chain + /// whose gate is off. It is a *build* fact, not a property of the input: + /// the validation code was not compiled, so there is no answer to give. + /// Reporting it as an error rather than silently accepting or rejecting + /// the address is the point — either of those would be a wrong answer + /// dressed up as a real one. + #[error( + "tinywallet was built without support for {chain}; \ + enable the '{chain}' feature to validate its addresses" + )] + ChainNotCompiled { + /// The chain whose feature gate is disabled. + chain: Chain, + }, +} + +/// The crate's standard result type. +/// +/// Use this alias in public signatures instead of spelling out +/// `std::result::Result`. +pub type Result = std::result::Result; + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/error/test.rs b/src/openhuman/web3/wallet/primitives/error/test.rs new file mode 100644 index 0000000000..2f5417d9b0 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/error/test.rs @@ -0,0 +1,63 @@ +//! Unit tests for the crate-wide error type. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::Error; +use crate::openhuman::web3::wallet::primitives::Chain; + +#[test] +fn empty_address_names_its_chain() { + let err = Error::EmptyAddress { chain: Chain::Btc }; + assert_eq!(err.to_string(), "btc address is empty"); +} + +#[test] +fn invalid_address_shows_the_address_and_the_reason() { + // The rejected address appears verbatim: it is public data, and eliding it + // turns a one-line fix into a debugging session. + let err = Error::InvalidAddress { + chain: Chain::Solana, + address: "0OIl".to_string(), + reason: "not valid base58".to_string(), + }; + let rendered = err.to_string(); + assert!(rendered.contains("0OIl"), "{rendered}"); + assert!(rendered.contains("not valid base58"), "{rendered}"); + assert!(rendered.contains("solana"), "{rendered}"); +} + +#[test] +fn wrong_network_names_the_expected_network() { + let err = Error::WrongNetwork { + chain: Chain::Btc, + address: "tb1qexample".to_string(), + expected: "mainnet".to_string(), + reason: "address is testnet".to_string(), + }; + assert!(err.to_string().contains("mainnet")); +} + +#[test] +fn unsupported_address_type_explains_what_is_supported() { + let err = Error::UnsupportedAddressType { + chain: Chain::Btc, + address: "1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2".to_string(), + reason: "only P2WPKH (bc1q… native segwit) can be signed for".to_string(), + }; + // A caller reading this should learn the fix, not just the failure. + assert!(err.to_string().contains("P2WPKH")); +} + +#[test] +fn errors_compare_by_value() { + // Callers assert on specific errors in their own tests, so equality has to + // be structural rather than by message. + assert_eq!( + Error::EmptyAddress { chain: Chain::Evm }, + Error::EmptyAddress { chain: Chain::Evm } + ); + assert_ne!( + Error::EmptyAddress { chain: Chain::Evm }, + Error::EmptyAddress { chain: Chain::Btc } + ); +} diff --git a/src/openhuman/web3/wallet/primitives/key/bip32.rs b/src/openhuman/web3/wallet/primitives/key/bip32.rs new file mode 100644 index 0000000000..2981811cd0 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/key/bip32.rs @@ -0,0 +1,114 @@ +//! BIP-32 derivation on secp256k1, shared by Bitcoin, EVM and Tron. +//! +//! All three chains use the same scheme and differ only in what they do with +//! the resulting key, so the walk lives here once rather than three times. +//! +//! # This is delegated on purpose +//! +//! Rolling BIP-32 by hand is possible — it is HMAC-SHA512 plus a scalar +//! addition — but an off-by-one in the hardened-index encoding produces a +//! *valid key for the wrong account*, which is silent, unrecoverable, and +//! exactly the kind of bug not worth risking to avoid a dependency. +//! +//! That reasoning is unchanged from when this used `bitcoin`'s `Xpriv`. What +//! changed is which vetted implementation it delegates to: `coins-bip32`, whose +//! secp256k1 backend is the pure-Rust `k256` rather than the `secp256k1` C +//! library. The derived key is identical either way — BIP-32 is a specification, +//! not an implementation detail, and [`super::test`] pins the addresses against +//! the same fixed mnemonic as before the swap. `coins-bip32` is also the code +//! path `coins-bip39` already uses beneath [`super::seed_from_mnemonic`], so +//! this removes a native C build and a second elliptic-curve stack without +//! adding anything to the graph. +//! +//! Contrast [`crate::openhuman::web3::wallet::primitives::address::btc`], which *is* hand-rolled. The difference is +//! the failure mode, not the difficulty: a wrong parser is caught by the first +//! test vector, a wrong derivation is caught by nobody. + +use std::str::FromStr; + +use coins_bip32::path::DerivationPath; +use coins_bip32::prelude::SigningKey; +use coins_bip32::xkeys::XPriv; + +use super::{Error, Result}; + +/// A secp256k1 key derived at a BIP-32 path. +pub(super) struct Secp256k1Key { + pub(super) secret: SigningKey, +} + +impl Secp256k1Key { + /// The 65-byte uncompressed SEC1 encoding, `0x04` prefix included. + /// + /// EVM and Tron both hash this — minus the prefix byte — with Keccak-256 to + /// form an address. + pub(super) fn uncompressed_public(&self) -> [u8; 65] { + let encoded = self.secret.verifying_key().to_encoded_point(false); + let mut out = [0u8; 65]; + // Uncompressed SEC1 is 65 bytes by definition, so this cannot be short. + out.copy_from_slice(encoded.as_bytes()); + out + } + + /// The 33-byte compressed SEC1 encoding. + /// + /// Bitcoin hashes this — not the uncompressed form — to form a P2WPKH + /// address. Using the wrong one yields a well-formed address for an account + /// nobody holds the key to, which is why the two encodings are separate + /// named methods rather than one with a boolean. + pub(super) fn compressed_public(&self) -> [u8; 33] { + let encoded = self.secret.verifying_key().to_encoded_point(true); + let mut out = [0u8; 33]; + out.copy_from_slice(encoded.as_bytes()); + out + } + + /// The 32-byte secret scalar. + pub(super) fn secret_bytes(&self) -> [u8; 32] { + self.secret.to_bytes().into() + } +} + +/// Walk `path` from the master key for `seed`. +/// +/// The derived secret does not depend on a network: BIP-32 version bytes only +/// matter when an extended key is serialized, which never happens here. The +/// same walk is therefore correct for Bitcoin, EVM and Tron alike. +pub(super) fn derive(seed: &[u8], path: &str) -> Result { + let master = XPriv::root_from_seed(seed, None).map_err(|_| Error::Derivation { + step: "BIP-32 master key", + })?; + let parsed = DerivationPath::from_str(path).map_err(|e| Error::InvalidPath { + path: path.to_string(), + reason: e.to_string(), + })?; + + // Depth is checked here rather than left to the backend, because + // `coins-bip32` does not check it: `derive_child` increments a `u8` depth + // unguarded, which panics in a debug build and **wraps silently in a + // release build** — deriving at a wrapped depth instead of refusing. The + // `bitcoin` implementation this replaced returned `MaximumDepthExceeded`, + // so without this the swap would have traded a clean error for a wrong key. + // + // The master node is depth 0, leaving 255 usable levels. No real path comes + // close; the bound exists so a hostile or generated one cannot get through. + if parsed.len() > usize::from(u8::MAX) { + return Err(Error::InvalidPath { + path: path.to_string(), + reason: format!( + "BIP-32 depth is limited to {} levels, got {}", + u8::MAX, + parsed.len() + ), + }); + } + + let child = master.derive_path(parsed).map_err(|_| Error::Derivation { + step: "BIP-32 child key", + })?; + + let secret: &SigningKey = child.as_ref(); + Ok(Secp256k1Key { + secret: secret.clone(), + }) +} diff --git a/src/openhuman/web3/wallet/primitives/key/btc.rs b/src/openhuman/web3/wallet/primitives/key/btc.rs new file mode 100644 index 0000000000..ec632b5991 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/key/btc.rs @@ -0,0 +1,51 @@ +//! Bitcoin key derivation: BIP-32 on secp256k1, P2WPKH address. +//! +//! Produces a native segwit (`bc1q…`) address, matching +//! [`crate::openhuman::web3::wallet::primitives::address::btc::validate_sender`] — the only script type this crate's +//! callers can sign for. Deriving a P2PKH or P2SH address here would hand back +//! something that passes recipient validation and then fails at signing time. +//! +//! The address is assembled here rather than by the `bitcoin` crate, which this +//! module used to route through. A P2WPKH address is fully specified by BIP-141 +//! and BIP-173 as `bech32(hrp="bc", version=0, hash160(compressed_pubkey))`, and +//! both halves of that are owned elsewhere: the bech32 encoding by +//! [`crate::openhuman::web3::wallet::primitives::address::btc::encode_p2wpkh`], which also decodes it, and the +//! BIP-32 walk by [`super::bip32`], which still delegates to a vetted +//! implementation. + +use ripemd::Ripemd160; +use sha2::{Digest, Sha256}; + +use super::{bip32, seed_from_mnemonic, DerivedKey, Error, Result}; +use crate::openhuman::web3::wallet::primitives::address::btc::encode_p2wpkh; +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// Derive the Bitcoin signing key and P2WPKH address for `path`. +pub(super) fn derive(mnemonic: &str, path: &str) -> Result { + let seed = seed_from_mnemonic(mnemonic)?; + let key = bip32::derive(&seed, path)?; + + // The *compressed* encoding: a P2WPKH witness program is defined over it, + // and hashing the uncompressed form instead produces a valid-looking + // address for an account holding no funds. + let address = + encode_p2wpkh(&hash160(&key.compressed_public())).map_err(|_| Error::Derivation { + step: "BTC P2WPKH address", + })?; + + Ok(DerivedKey::new( + Chain::Btc, + address, + key.secret_bytes().to_vec(), + )) +} + +/// `RIPEMD160(SHA256(data))` — Bitcoin's HASH160. +fn hash160(data: &[u8]) -> [u8; 20] { + let sha = Sha256::digest(data); + let ripemd = Ripemd160::digest(sha); + let mut out = [0u8; 20]; + // RIPEMD-160 is 20 bytes by definition. + out.copy_from_slice(&ripemd); + out +} diff --git a/src/openhuman/web3/wallet/primitives/key/evm.rs b/src/openhuman/web3/wallet/primitives/key/evm.rs new file mode 100644 index 0000000000..22fb22293d --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/key/evm.rs @@ -0,0 +1,40 @@ +//! EVM key derivation: BIP-32 on secp256k1, address via Keccak-256. + +use sha3::{Digest, Keccak256}; + +use super::{bip32, seed_from_mnemonic, DerivedKey, Result}; +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// Derive the EVM signing key and address for `path`. +pub(super) fn derive(mnemonic: &str, path: &str) -> Result { + let seed = seed_from_mnemonic(mnemonic)?; + let key = bip32::derive(&seed, path)?; + let address = address_from_public(&key.uncompressed_public()); + Ok(DerivedKey::new( + Chain::Evm, + address, + key.secret_bytes().to_vec(), + )) +} + +/// An EVM address is the last 20 bytes of the Keccak-256 hash of the +/// uncompressed public key with its `0x04` prefix byte removed. +/// +/// Returned EIP-55 checksummed, which is the canonical display form and what +/// every explorer and wallet shows. +fn address_from_public(uncompressed: &[u8; 65]) -> String { + let hash = Keccak256::digest(&uncompressed[1..]); + let body = hex_lower(&hash[12..]); + // The address was just built from a hash, so it is well-formed by + // construction and checksumming cannot fail. + crate::openhuman::web3::wallet::primitives::address::evm::to_checksummed(&body) + .unwrap_or_else(|_| format!("0x{body}")) +} + +fn hex_lower(bytes: &[u8]) -> String { + use std::fmt::Write as _; + bytes.iter().fold(String::new(), |mut out, b| { + let _ = write!(out, "{b:02x}"); + out + }) +} diff --git a/src/openhuman/web3/wallet/primitives/key/mod.rs b/src/openhuman/web3/wallet/primitives/key/mod.rs new file mode 100644 index 0000000000..78ae6dfd88 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/key/mod.rs @@ -0,0 +1,248 @@ +//! Deterministic key derivation from a BIP-39 mnemonic. +//! +//! [`derive()`] turns a mnemonic and a derivation path into the signing key and +//! address for one chain. It is a pure function: same inputs, same key, every +//! time, with no I/O and no global state. +//! +//! ## This crate derives keys; it does not keep them +//! +//! Nothing here reads or writes a keychain, a file, or an environment +//! variable, and no key is cached between calls. Custody is deliberately the +//! host's problem: where the mnemonic is sealed, what unlocks it, whether the +//! user is prompted, and how long a decrypted phrase may live in memory are +//! all policy decisions that depend on the host's threat model, and a library +//! that quietly picked an answer would be picking it for every host. +//! +//! The consequence for a caller is that the mnemonic arrives as a `&str` the +//! host already decrypted, and this crate's job is to touch it briefly and +//! forget it. +//! +//! ## Two derivation algorithms, not one +//! +//! | Chain | Curve | Scheme | +//! | --- | --- | --- | +//! | Bitcoin | secp256k1 | BIP-32 | +//! | EVM | secp256k1 | BIP-32 | +//! | Tron | secp256k1 | BIP-32 | +//! | Solana | ed25519 | SLIP-0010, hardened-only | +//! +//! The split is forced by the curve. BIP-32's non-hardened derivation needs +//! public-key addition, which ed25519 does not offer, so SLIP-0010 defines +//! hardened-only derivation for it. That is why [`Error::UnhardenedSolanaPath`] +//! exists: a path like `m/44'/501'/0'/0` is not merely unsupported here, it is +//! underivable, and accepting it by silently hardening the last segment would +//! hand back a *different account* than the path names. + +use zeroize::Zeroizing; + +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +mod bip32; +mod slip10; + +#[cfg(feature = "web3")] +mod btc; +#[cfg(feature = "web3")] +mod evm; +#[cfg(feature = "web3")] +mod solana; +#[cfg(feature = "web3")] +mod tron; + +/// Errors raised while deriving a key. +/// +/// Every variant names the failing *step*. None carries key material, a seed, +/// or any part of a mnemonic — an error string is the single easiest way for a +/// secret to escape into a log, so nothing secret is ever put in one. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The mnemonic is not a valid BIP-39 phrase — wrong word count, a word + /// outside the wordlist, or a failed checksum. + /// + /// Deliberately carries no detail beyond this. The underlying error can + /// quote the offending word, which is one twelfth of a seed phrase. + #[error("invalid BIP-39 mnemonic")] + InvalidMnemonic, + + /// The derivation path is not well-formed. + #[error("invalid derivation path '{path}': {reason}")] + InvalidPath { + /// The rejected path. A path is not secret — it is public metadata + /// about which account was meant. + path: String, + /// Why it was rejected. + reason: String, + }, + + /// A Solana path contains a non-hardened segment. + /// + /// Separate from [`Error::InvalidPath`] because it is not a typo: the path + /// is syntactically fine and simply cannot be derived on ed25519. See the + /// module docs — silently hardening it would return a different account + /// than the caller asked for. + #[error( + "Solana path '{path}' has a non-hardened segment; ed25519 (SLIP-0010) \ + supports hardened derivation only, so every segment needs a trailing '" + )] + UnhardenedSolanaPath { + /// The rejected path. + path: String, + }, + + /// Key derivation failed arithmetically. + /// + /// Essentially unreachable in practice: BIP-32 specifies retrying with the + /// next index when a derived scalar falls outside the curve order, and the + /// odds of hitting that are negligible. It is a variant rather than a panic + /// because a wallet must not abort the process over it. + #[error("key derivation failed at {step}")] + Derivation { + /// Which step failed. + step: &'static str, + }, + + /// The chain's feature gate was disabled when this crate was built. + /// + /// A build fact, not a property of the inputs — the same reasoning as + /// [`crate::openhuman::web3::wallet::primitives::Error::ChainNotCompiled`]. + #[error( + "tinywallet was built without support for {chain}; \ + enable the '{chain}' feature to derive its keys" + )] + ChainNotCompiled { + /// The chain whose gate is disabled. + chain: Chain, + }, +} + +/// Result alias for key derivation. +pub type Result = std::result::Result; + +/// A derived signing key and the address it controls. +/// +/// The secret is held in [`Zeroizing`], so dropping this wipes it rather than +/// leaving it in freed memory for whatever allocates there next. +/// +/// `Debug` is implemented by hand and prints only the chain and address. +/// Deriving it would put raw key material into every `{:?}`, every +/// `unwrap()` panic message, and every log line that formats a struct +/// containing one — which is exactly how a private key ends up in a bug +/// report. +pub struct DerivedKey { + chain: Chain, + address: String, + secret: Zeroizing>, +} + +impl DerivedKey { + /// Build a derived key. Internal: the per-chain modules construct these. + fn new(chain: Chain, address: String, secret: Vec) -> Self { + Self { + chain, + address, + secret: Zeroizing::new(secret), + } + } + + /// The chain this key is for. + #[must_use] + pub const fn chain(&self) -> Chain { + self.chain + } + + /// The address this key controls, in the chain's canonical text form. + #[must_use] + pub fn address(&self) -> &str { + &self.address + } + + /// The raw secret key bytes. + /// + /// 32 bytes on every supported chain. Treat the returned slice as live key + /// material: do not copy it into a `String`, a log, or an error. It is + /// borrowed rather than returned by value so it cannot outlive the + /// zeroizing owner. + #[must_use] + pub fn secret_bytes(&self) -> &[u8] { + &self.secret + } +} + +impl std::fmt::Debug for DerivedKey { + /// Prints the chain and address only. See the type docs: a derived `Debug` + /// here would leak key material into panic messages and logs. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DerivedKey") + .field("chain", &self.chain) + .field("address", &self.address) + .field("secret", &"") + .finish() + } +} + +/// Derive the signing key and address for `chain` from `mnemonic` at `path`. +/// +/// `mnemonic` is a BIP-39 phrase the host has already decrypted; it is used +/// for the duration of the call and not retained. `path` is a BIP-32 style +/// derivation path (`m/44'/60'/0'/0/0`). +/// +/// # Errors +/// +/// - [`Error::InvalidMnemonic`] if the phrase is not valid BIP-39. +/// - [`Error::InvalidPath`] if the path is malformed. +/// - [`Error::UnhardenedSolanaPath`] for a Solana path with a non-hardened +/// segment — see the module docs for why that is its own variant. +/// - [`Error::ChainNotCompiled`] if `chain`'s feature gate is off. +/// +/// # Examples +/// +/// ``` +/// # #[cfg(feature = "web3")] { +/// use crate::openhuman::web3::wallet::primitives::{key, Chain}; +/// +/// // The BIP-39 test vector mnemonic. Never use it for real funds. +/// let phrase = "abandon abandon abandon abandon abandon abandon \ +/// abandon abandon abandon abandon abandon about"; +/// let derived = key::derive(Chain::Evm, phrase, "m/44'/60'/0'/0/0")?; +/// +/// assert_eq!(derived.address(), "0x9858EfFD232B4033E47d90003D41EC34EcaEda94"); +/// // Debug never prints the secret. +/// assert!(format!("{derived:?}").contains("")); +/// # } +/// # Ok::<(), crate::openhuman::web3::wallet::primitives::key::Error>(()) +/// ``` +pub fn derive(chain: Chain, mnemonic: &str, path: &str) -> Result { + match chain { + #[cfg(feature = "web3")] + Chain::Btc => btc::derive(mnemonic, path), + #[cfg(feature = "web3")] + Chain::Evm => evm::derive(mnemonic, path), + #[cfg(feature = "web3")] + Chain::Solana => solana::derive(mnemonic, path), + #[cfg(feature = "web3")] + Chain::Tron => tron::derive(mnemonic, path), + #[allow(unreachable_patterns)] + other => Err(Error::ChainNotCompiled { chain: other }), + } +} + +/// Turn a BIP-39 phrase into its 64-byte seed. +/// +/// Shared by every chain: the seed is scheme-independent, and only what +/// happens after it differs. The result zeroizes on drop. +fn seed_from_mnemonic(mnemonic: &str) -> Result>> { + use coins_bip39::{English, Mnemonic}; + + // The error is discarded on purpose: `coins_bip39` reports which word + // failed the wordlist check, and a word is one twelfth of a seed phrase. + let parsed: Mnemonic = mnemonic + .trim() + .parse() + .map_err(|_| Error::InvalidMnemonic)?; + let seed = parsed.to_seed(None).map_err(|_| Error::InvalidMnemonic)?; + Ok(Zeroizing::new(seed.to_vec())) +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/key/slip10.rs b/src/openhuman/web3/wallet/primitives/key/slip10.rs new file mode 100644 index 0000000000..dfea18d1b5 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/key/slip10.rs @@ -0,0 +1,105 @@ +//! SLIP-0010 hardened-only derivation on ed25519, used by Solana. +//! +//! ed25519 cannot do BIP-32's non-hardened derivation: that step needs +//! public-key addition, which the curve's key format does not support. SLIP-0010 +//! defines the hardened-only variant instead, and this is it — about twenty +//! lines of HMAC-SHA512, with no scalar arithmetic and so no failure mode +//! beyond a malformed path. +//! +//! Solana wallets standardise on `m/44'/501'/N'/0'`, which is fully hardened, +//! so the restriction costs nothing in practice. + +use hmac::{Hmac, Mac}; +use sha2::Sha512; +use zeroize::Zeroizing; + +use super::{Error, Result}; + +type HmacSha512 = Hmac; + +/// The domain-separation key SLIP-0010 specifies for ed25519. +const CURVE_SEED: &[u8] = b"ed25519 seed"; + +/// Derive the 32-byte ed25519 secret for `path` from `seed`. +/// +/// `path` must be fully hardened. Each index is OR-ed with `0x8000_0000` +/// regardless, but [`parse_path`] rejects an unhardened segment first — see +/// [`Error::UnhardenedSolanaPath`] for why that is not silently tolerated. +pub(super) fn derive(seed: &[u8], path: &str) -> Result> { + let indices = parse_path(path)?; + + let mut mac = HmacSha512::new_from_slice(CURVE_SEED).map_err(|_| Error::Derivation { + step: "SLIP-0010 master HMAC", + })?; + mac.update(seed); + let digest = mac.finalize().into_bytes(); + + let mut key = Zeroizing::new([0u8; 32]); + let mut chain_code = Zeroizing::new([0u8; 32]); + key.copy_from_slice(&digest[..32]); + chain_code.copy_from_slice(&digest[32..]); + + for index in indices { + let hardened = index | 0x8000_0000; + let mut mac = + HmacSha512::new_from_slice(chain_code.as_slice()).map_err(|_| Error::Derivation { + step: "SLIP-0010 child HMAC", + })?; + // The leading zero byte is what marks this as the hardened form. + mac.update(&[0u8]); + mac.update(key.as_slice()); + mac.update(&hardened.to_be_bytes()); + let digest = mac.finalize().into_bytes(); + key.copy_from_slice(&digest[..32]); + chain_code.copy_from_slice(&digest[32..]); + } + + Ok(key) +} + +/// Parse a fully hardened path into its indices. +/// +/// # Errors +/// +/// [`Error::InvalidPath`] if the path does not start at `m`, has no segments, +/// or holds a non-numeric index. [`Error::UnhardenedSolanaPath`] if any segment +/// lacks its trailing apostrophe. +fn parse_path(path: &str) -> Result> { + let trimmed = path.trim(); + let mut segments = trimmed.split('/'); + + if segments.next() != Some("m") { + return Err(Error::InvalidPath { + path: path.to_string(), + reason: "must start with 'm'".to_string(), + }); + } + + let mut out = Vec::new(); + for segment in segments { + let Some(index) = segment.strip_suffix('\'') else { + return Err(Error::UnhardenedSolanaPath { + path: path.to_string(), + }); + }; + let index = index.parse::().map_err(|e| Error::InvalidPath { + path: path.to_string(), + reason: format!("segment '{segment}': {e}"), + })?; + if index >= 0x8000_0000 { + return Err(Error::InvalidPath { + path: path.to_string(), + reason: format!("segment '{segment}' exceeds the maximum raw index"), + }); + } + out.push(index); + } + + if out.is_empty() { + return Err(Error::InvalidPath { + path: path.to_string(), + reason: "has no segments".to_string(), + }); + } + Ok(out) +} diff --git a/src/openhuman/web3/wallet/primitives/key/solana.rs b/src/openhuman/web3/wallet/primitives/key/solana.rs new file mode 100644 index 0000000000..4ca8b729a2 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/key/solana.rs @@ -0,0 +1,20 @@ +//! Solana key derivation: SLIP-0010 on ed25519, address is the public key. + +use ed25519_dalek::SigningKey; + +use super::{seed_from_mnemonic, slip10, DerivedKey, Result}; +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// Derive the Solana signing key and address for `path`. +/// +/// A Solana address *is* the ed25519 public key in base58 — there is no hash +/// and no version byte, unlike every other chain here. +pub(super) fn derive(mnemonic: &str, path: &str) -> Result { + let seed = seed_from_mnemonic(mnemonic)?; + let secret = slip10::derive(&seed, path)?; + let signing = SigningKey::from_bytes(&secret); + let address = crate::openhuman::web3::wallet::primitives::address::solana::encode( + &signing.verifying_key().to_bytes(), + ); + Ok(DerivedKey::new(Chain::Solana, address, secret.to_vec())) +} diff --git a/src/openhuman/web3/wallet/primitives/key/test.rs b/src/openhuman/web3/wallet/primitives/key/test.rs new file mode 100644 index 0000000000..efe8914c62 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/key/test.rs @@ -0,0 +1,294 @@ +//! Unit tests for key derivation. +//! +//! These are pinned against the canonical BIP-39 test-vector mnemonic and the +//! addresses every mainstream wallet derives from it. That matters more here +//! than in most test suites: a derivation bug does not crash, it produces a +//! *valid key for the wrong account*, and the only way to catch that is to +//! compare against an address derived independently by other software. +//! +//! The mnemonic below is the published all-`abandon` test vector. It is public, +//! and its accounts have been swept continuously for years — never put funds +//! in an address derived from it. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::{derive, Error}; +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// The canonical BIP-39 test vector: 11 × "abandon" + "about". +const VECTOR: &str = "abandon abandon abandon abandon abandon abandon \ + abandon abandon abandon abandon abandon about"; + +/// Standard first-account path per chain, matching each ecosystem's default. +const EVM_PATH: &str = "m/44'/60'/0'/0/0"; +const BTC_PATH: &str = "m/84'/0'/0'/0/0"; +const TRON_PATH: &str = "m/44'/195'/0'/0/0"; +const SOLANA_PATH: &str = "m/44'/501'/0'/0'"; + +#[test] +fn evm_matches_the_published_test_vector() { + // This is the address MetaMask, Trust and every EIP-55 tool derive from + // the vector mnemonic at the standard Ethereum path. + let key = derive(Chain::Evm, VECTOR, EVM_PATH).unwrap(); + assert_eq!(key.address(), "0x9858EfFD232B4033E47d90003D41EC34EcaEda94"); + assert_eq!(key.chain(), Chain::Evm); + assert_eq!(key.secret_bytes().len(), 32); +} + +#[test] +fn btc_derives_the_published_native_segwit_vector() { + // BIP-84's own test vector for account 0, first receive address. + let key = derive(Chain::Btc, VECTOR, BTC_PATH).unwrap(); + assert_eq!(key.address(), "bc1qcr8te4kr609gcawutmrza0j4xv80jy8z306fyu"); + assert!( + key.address().starts_with("bc1q"), + "must be P2WPKH — the only type this crate can sign for" + ); +} + +#[test] +fn btc_derives_an_address_its_own_sender_rule_accepts() { + // The derived address has to satisfy `validate_sender`, not merely + // `validate`. Deriving a P2PKH here would pass recipient validation and + // then fail at signing time. + let key = derive(Chain::Btc, VECTOR, BTC_PATH).unwrap(); + assert!( + crate::openhuman::web3::wallet::primitives::address::btc::validate_sender(key.address()) + .is_ok() + ); +} + +#[test] +fn solana_derives_the_published_vector() { + let key = derive(Chain::Solana, VECTOR, SOLANA_PATH).unwrap(); + assert_eq!( + key.address(), + "HAgk14JpMQLgt6rVgv7cBQFJWFto5Dqxi472uT3DKpqk" + ); + assert_eq!(key.chain(), Chain::Solana); +} + +#[test] +fn every_chain_derives_an_address_its_own_validator_accepts() { + // Cheap end-to-end coupling check between `key` and `address`: a + // derivation that produced a malformed address would be caught here even + // without a published vector to compare against. + for (chain, path) in [ + (Chain::Evm, EVM_PATH), + (Chain::Btc, BTC_PATH), + (Chain::Tron, TRON_PATH), + (Chain::Solana, SOLANA_PATH), + ] { + let key = derive(chain, VECTOR, path).unwrap(); + assert!( + crate::openhuman::web3::wallet::primitives::address::validate(chain, key.address()) + .is_ok(), + "{chain} derived an address its own validator rejects: {}", + key.address() + ); + assert_eq!(key.chain(), chain); + assert_eq!(key.secret_bytes().len(), 32, "{chain} secret length"); + } +} + +#[test] +fn tron_derives_a_mainnet_address_not_an_evm_one() { + // Tron reuses Ethereum's address construction then re-encodes it, so the + // easy bug is emitting the 20-byte EVM form. It must be 21 bytes with the + // 0x41 version prefix, in base58check. + let key = derive(Chain::Tron, VECTOR, TRON_PATH).unwrap(); + assert!( + key.address().starts_with('T'), + "expected base58check Tron form, got {}", + key.address() + ); + let decoded = + crate::openhuman::web3::wallet::primitives::address::tron::decode(key.address()).unwrap(); + assert_eq!(decoded.len(), 21); + assert_eq!( + decoded[0], + crate::openhuman::web3::wallet::primitives::address::tron::MAINNET_PREFIX + ); +} + +#[test] +fn evm_and_tron_share_a_key_but_not_an_address() { + // Both are secp256k1 + Keccak, so at the same path the secret is identical + // and only the encoding differs. Pinning this documents why Tron support + // costs almost nothing beyond an encoder. + let evm = derive(Chain::Evm, VECTOR, EVM_PATH).unwrap(); + let tron = derive(Chain::Tron, VECTOR, EVM_PATH).unwrap(); + assert_eq!(evm.secret_bytes(), tron.secret_bytes()); + assert_ne!(evm.address(), tron.address()); +} + +#[test] +fn derivation_is_deterministic() { + for (chain, path) in [(Chain::Evm, EVM_PATH), (Chain::Solana, SOLANA_PATH)] { + let first = derive(chain, VECTOR, path).unwrap(); + let second = derive(chain, VECTOR, path).unwrap(); + assert_eq!(first.address(), second.address()); + assert_eq!(first.secret_bytes(), second.secret_bytes()); + } +} + +#[test] +fn a_different_path_yields_a_different_account() { + let first = derive(Chain::Evm, VECTOR, "m/44'/60'/0'/0/0").unwrap(); + let second = derive(Chain::Evm, VECTOR, "m/44'/60'/0'/0/1").unwrap(); + assert_ne!(first.address(), second.address()); + assert_ne!(first.secret_bytes(), second.secret_bytes()); +} + +#[test] +fn the_mnemonic_is_trimmed_not_rejected_for_surrounding_whitespace() { + let padded = format!(" {VECTOR}\n"); + let key = derive(Chain::Evm, &padded, EVM_PATH).unwrap(); + assert_eq!(key.address(), "0x9858EfFD232B4033E47d90003D41EC34EcaEda94"); +} + +#[test] +fn an_invalid_mnemonic_is_rejected_without_quoting_it() { + // The error must not echo any part of the phrase — an error string is the + // easiest way for a secret to reach a log. + let bad = "abandon abandon notaword abandon abandon abandon \ + abandon abandon abandon abandon abandon about"; + let err = derive(Chain::Evm, bad, EVM_PATH).unwrap_err(); + assert_eq!(err, Error::InvalidMnemonic); + let rendered = err.to_string(); + assert!(!rendered.contains("notaword"), "leaked a word: {rendered}"); + assert!(!rendered.contains("abandon"), "leaked a word: {rendered}"); +} + +#[test] +fn a_wrong_length_mnemonic_is_rejected() { + assert_eq!( + derive(Chain::Evm, "abandon about", EVM_PATH).unwrap_err(), + Error::InvalidMnemonic + ); +} + +#[test] +fn a_malformed_path_is_rejected_and_names_the_path() { + // A path is public metadata about which account was meant, so unlike the + // mnemonic it is safe — and useful — to echo. + match derive(Chain::Evm, VECTOR, "not-a-path").unwrap_err() { + Error::InvalidPath { path, .. } => assert_eq!(path, "not-a-path"), + other => panic!("expected InvalidPath, got {other:?}"), + } +} + +#[test] +fn an_unhardened_solana_path_is_rejected_rather_than_silently_hardened() { + // The heart of the SLIP-0010 restriction: this path is syntactically fine + // and simply cannot be derived on ed25519. Hardening it silently would + // return a DIFFERENT account than the caller named, which is the failure + // this variant exists to prevent. + match derive(Chain::Solana, VECTOR, "m/44'/501'/0'/0").unwrap_err() { + Error::UnhardenedSolanaPath { path } => assert_eq!(path, "m/44'/501'/0'/0"), + other => panic!("expected UnhardenedSolanaPath, got {other:?}"), + } +} + +#[test] +fn a_solana_path_with_no_segments_is_rejected() { + assert!(matches!( + derive(Chain::Solana, VECTOR, "m").unwrap_err(), + Error::InvalidPath { .. } + )); +} + +#[test] +fn a_solana_path_not_starting_at_m_is_rejected() { + assert!(matches!( + derive(Chain::Solana, VECTOR, "44'/501'/0'/0'").unwrap_err(), + Error::InvalidPath { .. } + )); +} + +#[test] +fn a_solana_path_with_a_non_numeric_segment_is_rejected() { + match derive(Chain::Solana, VECTOR, "m/44'/not-an-index'/0'").unwrap_err() { + Error::InvalidPath { path, reason } => { + assert_eq!(path, "m/44'/not-an-index'/0'"); + assert!(reason.contains("not-an-index"), "{reason}"); + } + other => panic!("expected InvalidPath, got {other:?}"), + } +} + +#[test] +fn a_solana_path_with_an_already_hardened_index_is_rejected() { + assert!(matches!( + derive(Chain::Solana, VECTOR, "m/44'/501'/2147483648'").unwrap_err(), + Error::InvalidPath { .. } + )); +} + +#[test] +fn derivation_backend_failures_remain_specific_without_leaking_inputs() { + // Drives the real derivation path rather than the backend's error mapper. + // The previous version of this test called two private helpers with a + // hand-built `bitcoin::bip32::Error`; both are gone, and one of them — + // the uncompressed-public-key mapper — no longer has a reachable failure + // mode at all, because the address is now encoded from the compressed + // SEC1 point directly. Asserting on behaviour instead means this test + // survives the next backend swap the way it did not survive this one. + // + // BIP-32 depth is a single byte, so a path past 255 levels cannot be + // walked. This must be a clean refusal: the `coins-bip32` backend + // increments its depth counter unguarded, so without tinywallet's own + // bound this input panics in debug and — far worse — silently wraps in + // release, deriving a real key at the wrong depth. + let too_deep = format!("m/{}", vec!["0"; 256].join("/")); + let error = derive(Chain::Btc, VECTOR, &too_deep).unwrap_err(); + + match &error { + Error::InvalidPath { path, reason } => { + assert_eq!(path, &too_deep); + assert!(reason.contains("255"), "{reason}"); + } + other => panic!("expected InvalidPath for an over-deep path, got {other:?}"), + } + + // The depth just under the limit must still derive, so the bound is a + // guard rather than an off-by-one that rejects legitimate paths. + let deepest = format!("m/{}", vec!["0"; 255].join("/")); + assert!( + derive(Chain::Btc, VECTOR, &deepest).is_ok(), + "255 levels is the documented maximum and must still derive" + ); + + // The whole point of collapsing backend errors into a fixed `step` string: + // the mnemonic and the path must not ride out inside the message. + let rendered = error.to_string(); + for secret in VECTOR.split_whitespace() { + assert!( + !rendered.contains(secret), + "derivation error leaked mnemonic word '{secret}': {rendered}" + ); + } +} + +#[test] +fn debug_never_prints_key_material() { + // A derived Debug here would put a private key into every panic message + // and every log line that formats a struct containing one. + let key = derive(Chain::Evm, VECTOR, EVM_PATH).unwrap(); + let rendered = format!("{key:?}"); + + assert!(rendered.contains(""), "{rendered}"); + assert!(rendered.contains(key.address()), "address is safe to show"); + + // The secret must not appear in any plausible encoding. + let hex = key.secret_bytes().iter().fold(String::new(), |mut out, b| { + use std::fmt::Write as _; + let _ = write!(out, "{b:02x}"); + out + }); + assert!(!rendered.contains(&hex), "leaked the secret as hex"); + assert!( + !rendered.contains(&format!("{:?}", key.secret_bytes())), + "leaked the secret as a byte slice" + ); +} diff --git a/src/openhuman/web3/wallet/primitives/key/tron.rs b/src/openhuman/web3/wallet/primitives/key/tron.rs new file mode 100644 index 0000000000..9105dc8613 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/key/tron.rs @@ -0,0 +1,44 @@ +//! Tron key derivation: BIP-32 on secp256k1, address via Keccak-256 plus the +//! `0x41` version byte and a base58check envelope. +//! +//! Identical to EVM up to the Keccak hash — Tron reuses Ethereum's address +//! construction and then re-encodes it. That similarity is a trap worth naming: +//! the hex form of a Tron address looks like an EVM address but is 21 bytes, +//! not 20, because of the version prefix. + +use sha3::{Digest, Keccak256}; + +use super::{bip32, seed_from_mnemonic, DerivedKey, Error, Result}; +use crate::openhuman::web3::wallet::primitives::address::tron::{ADDRESS_BYTES, MAINNET_PREFIX}; +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// Derive the Tron signing key and address for `path`. +pub(super) fn derive(mnemonic: &str, path: &str) -> Result { + let seed = seed_from_mnemonic(mnemonic)?; + let key = bip32::derive(&seed, path)?; + let address = address_from_public(&key.uncompressed_public())?; + Ok(DerivedKey::new( + Chain::Tron, + address, + key.secret_bytes().to_vec(), + )) +} + +/// Keccak-256 the uncompressed public key without its `0x04` prefix, take the +/// last 20 bytes, prepend the Tron mainnet version byte, and base58check it. +/// +/// `encode` verifies the version byte and so returns a `Result`. It cannot +/// fail here — the prefix is written two lines above — but the error is mapped +/// rather than unwrapped, because a panic in key derivation would take a +/// wallet down over an unreachable branch. +fn address_from_public(uncompressed: &[u8; 65]) -> Result { + let hash = Keccak256::digest(&uncompressed[1..]); + let mut bytes = [0u8; ADDRESS_BYTES]; + bytes[0] = MAINNET_PREFIX; + bytes[1..].copy_from_slice(&hash[12..]); + crate::openhuman::web3::wallet::primitives::address::tron::encode(&bytes).map_err(|_| { + Error::Derivation { + step: "Tron address encoding", + } + }) +} diff --git a/src/openhuman/web3/wallet/primitives/mod.rs b/src/openhuman/web3/wallet/primitives/mod.rs new file mode 100644 index 0000000000..793506e634 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/mod.rs @@ -0,0 +1,74 @@ +//! Agent-friendly multi-chain wallet primitives in Rust. +//! +//! `tinywallet` owns the parts of wallet handling that are pure: address +//! formats, their validation, and the conversions between their encodings. +//! Bitcoin, EVM chains, Solana, and Tron each get a module, and +//! [`address::validate`] dispatches across them for chain-generic callers. +//! +//! # What this crate deliberately does not do +//! +//! No network access, no RPC endpoints, no key storage, no transaction +//! broadcasting. Every function here is a deterministic pure function of its +//! arguments. +//! +//! That is the seam, not a gap. Endpoint selection, retry policy, and key +//! custody are things a host must own — they depend on its config, its threat +//! model, and its runtime — and a crate that guessed at any of them would be +//! wrong for every host that guessed differently. What is left is the part +//! that is genuinely the same everywhere, which is exactly what belongs in a +//! shared crate. +//! +//! # Example +//! +//! ``` +//! # #[cfg(all(feature = "web3", feature = "web3"))] { +//! use crate::openhuman::web3::wallet::primitives::{address, chain::Chain}; +//! +//! // Chain-generic dispatch. +//! let addr = address::validate(Chain::Btc, "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4")?; +//! +//! // Or reach for a chain's own module when you need more than validation. +//! let hex = address::tron::to_hex("TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t")?; +//! assert!(hex.starts_with("41")); +//! # } +//! # Ok::<(), crate::openhuman::web3::wallet::primitives::Error>(()) +//! ``` +//! +//! # Feature flags +//! +//! Every chain is a separate default-on gate, so a host that only needs one +//! chain does not pay for the others' parsers. +//! +//! | Feature | Default | Gates | +//! | --- | --- | --- | +//! | `btc` | on | Bitcoin addresses (pulls `bitcoin`) | +//! | `evm` | on | EVM addresses (no dependencies) | +//! | `solana` | on | Solana addresses (pulls `bs58`) | +//! | `tron` | on | Tron addresses (pulls `bs58`, `hex`) | +//! | `keccak` | on | EIP-55 checksums for EVM (pulls `sha3`) | +//! | `net` | on | the `rpc::Transport` network seam (pulls `async-trait`) | +//! | `key` | on | BIP-39/BIP-32/SLIP-0010 key derivation (`crate::openhuman::web3::wallet::primitives::key`) | +//! | `asset` | on | network and token reference data (`crate::openhuman::web3::wallet::primitives::asset`) | +//! | `client` | on | chain queries over the seam (`crate::openhuman::web3::wallet::primitives::client`) | +//! | `tx` | on | transaction building and signing (`crate::openhuman::web3::wallet::primitives::tx`) | +//! | `x402` | on | x402 machine-payment wire types (`crate::openhuman::web3::wallet::primitives::x402`) | + +mod error; + +#[cfg(feature = "web3")] +pub mod abi; +pub mod address; +pub mod chain; +#[cfg(feature = "web3")] +pub mod eip712; +#[cfg(feature = "web3")] +pub mod key; +#[cfg(feature = "web3")] +pub mod rpc; +#[cfg(feature = "web3")] +pub mod wire; +#[cfg(feature = "web3")] +pub mod x402; + +pub use chain::Chain; +pub use error::{Error, Result}; diff --git a/src/openhuman/web3/wallet/primitives/rpc/mod.rs b/src/openhuman/web3/wallet/primitives/rpc/mod.rs new file mode 100644 index 0000000000..1fc770e7cf --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/rpc/mod.rs @@ -0,0 +1,277 @@ +//! The network seam: [`Transport`], the trait a host implements so this crate +//! can reach a chain without owning an HTTP client. +//! +//! Everything else in `tinywallet` is a pure function. Chain work is not — a +//! balance, a nonce, a UTXO set and a broadcast all require a network round +//! trip — so the chain modules take a `&dyn Transport` and the host supplies +//! it. +//! +//! ## What the host keeps, and why the trait names a network rather than a URL +//! +//! No method here accepts a URL. That is the whole point of the seam: endpoint +//! selection is a host concern that this crate must not quietly take over. +//! A host typically resolves an endpoint from its own config, allows an +//! operator to override it per chain, fails over across an ordered list when +//! one is unreachable, and redacts the URL before it reaches a log. Every one +//! of those depends on the host's configuration and deployment, and a crate +//! that hardcoded even a default endpoint would silently route a user's +//! transactions through whichever provider this crate's author happened to +//! pick. +//! +//! So the division is: +//! +//! | This crate | The host | +//! | --- | --- | +//! | which RPC method, with which params | which endpoint answers it | +//! | how to encode and sign the payload | failover, retries, timeouts | +//! | what a response means | connection pooling, TLS, redaction in logs | +//! +//! ## Errors are split by retryability, and that distinction is load-bearing +//! +//! [`TransportError`] separates an endpoint being unreachable from a healthy +//! endpoint returning an authoritative error. A host that fails over across +//! endpoints must advance on the first and stop dead on the second: retrying a +//! genuine "insufficient funds" against three more endpoints yields the same +//! answer three more times, and — far worse — retrying an *ambiguous* failure +//! risks broadcasting a transaction twice. Collapsing the two into one error +//! type is how a failover loop turns a declined transaction into a +//! double-spend, so the distinction is in the type rather than left to a +//! string match on the message. + +use async_trait::async_trait; +use serde_json::Value; + +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// Identifies which network a request is bound for. +/// +/// A bare [`Chain`] is not enough for EVM: Ethereum, Base, Polygon and Arbitrum +/// share an address format and an RPC dialect but are different networks with +/// different endpoints. The EIP-155 chain id is the universal discriminator, so +/// it is what this carries — a host's own network enum does not have to leak +/// into this crate for it to say which network it meant. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct NetworkId { + /// The chain family. + pub chain: Chain, + /// EIP-155 chain id, for [`Chain::Evm`] only. + /// + /// `None` on every other chain, and on EVM when the caller genuinely means + /// "the host's default EVM network" rather than a specific one. + pub evm_chain_id: Option, +} + +impl NetworkId { + /// A non-EVM network, identified by its chain alone. + #[must_use] + pub const fn chain(chain: Chain) -> Self { + Self { + chain, + evm_chain_id: None, + } + } + + /// A specific EVM network, by EIP-155 chain id. + #[must_use] + pub const fn evm(chain_id: u64) -> Self { + Self { + chain: Chain::Evm, + evm_chain_id: Some(chain_id), + } + } +} + +impl std::fmt::Display for NetworkId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.evm_chain_id { + Some(id) => write!(f, "{}:{id}", self.chain), + None => write!(f, "{}", self.chain), + } + } +} + +/// A transport failure, split by whether retrying elsewhere could help. +/// +/// See the module docs: this distinction is what lets a host fail over safely, +/// and collapsing it is how a retry loop causes a double broadcast. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum TransportError { + /// The endpoint could not be reached or did not answer usefully — DNS + /// failure, connection refused, a timeout, a 5xx, an unparseable body. + /// + /// **Safe to retry against another endpoint** *for a read*. A host must + /// still not blindly retry a broadcast on this: a request that timed out + /// may well have been accepted. + #[error("transport failure contacting {network}: {message}")] + Unreachable { + /// The network the request was bound for. + network: NetworkId, + /// What went wrong. + message: String, + }, + + /// A healthy endpoint answered with an error — an invalid transaction, + /// insufficient funds, a rejected signature, a malformed request. + /// + /// **Never retry this elsewhere.** It is the network's real answer, and + /// another endpoint will give the same one. + #[error("{network} returned an error: {message}")] + Rpc { + /// The network that answered. + network: NetworkId, + /// The error the node reported. + message: String, + }, +} + +impl TransportError { + /// Whether trying a different endpoint could plausibly produce a different + /// answer. + /// + /// True only for [`TransportError::Unreachable`]. Note this answers + /// "could the *result* differ", not "is retrying safe" — a broadcast that + /// timed out may already have been accepted, so a host must decide that + /// separately. + #[must_use] + pub const fn is_retryable(&self) -> bool { + matches!(self, Self::Unreachable { .. }) + } + + /// The network this failure relates to. + #[must_use] + pub const fn network(&self) -> NetworkId { + match self { + Self::Unreachable { network, .. } | Self::Rpc { network, .. } => *network, + } + } +} + +/// Result alias for transport operations. +pub type TransportResult = std::result::Result; + +/// The network seam a host implements. +/// +/// Implementations are shared across concurrent chain operations, hence +/// `Send + Sync`. A host is expected to hold one long-lived HTTP client behind +/// this rather than building one per call, since rebuilding a TLS connector +/// per request also discards connection pooling. +/// +/// # Implementing +/// +/// ``` +/// use async_trait::async_trait; +/// use serde_json::Value; +/// use crate::openhuman::web3::wallet::primitives::rpc::{NetworkId, Transport, TransportError, TransportResult}; +/// +/// struct MyTransport; +/// +/// #[async_trait] +/// impl Transport for MyTransport { +/// async fn json_rpc( +/// &self, +/// network: NetworkId, +/// method: &str, +/// _params: Value, +/// ) -> TransportResult { +/// // Resolve `network` to an endpoint from your own config, POST the +/// // JSON-RPC envelope, and map a node-level `error` member onto +/// // TransportError::Rpc rather than Unreachable. +/// Err(TransportError::Unreachable { +/// network, +/// message: format!("{method}: not wired up"), +/// }) +/// } +/// +/// async fn rest_get(&self, network: NetworkId, path: &str) -> TransportResult { +/// Err(TransportError::Unreachable { network, message: path.to_string() }) +/// } +/// +/// async fn rest_post( +/// &self, +/// network: NetworkId, +/// path: &str, +/// _body: String, +/// _content_type: &str, +/// ) -> TransportResult { +/// Err(TransportError::Unreachable { network, message: path.to_string() }) +/// } +/// } +/// ``` +#[async_trait] +pub trait Transport: Send + Sync { + /// Perform a JSON-RPC call and return the `result` member. + /// + /// Used by EVM and Solana. The implementation wraps `method` and `params` + /// in the JSON-RPC envelope, sends it to whichever endpoint serves + /// `network`, and returns the `result` member on success. + /// + /// # Errors + /// + /// [`TransportError::Rpc`] when the node answers with an `error` member — + /// this is an authoritative answer and must not be retried elsewhere. + /// [`TransportError::Unreachable`] for anything that prevented getting an + /// answer at all. + async fn json_rpc( + &self, + network: NetworkId, + method: &str, + params: Value, + ) -> TransportResult; + + /// Perform a REST GET and return the raw body. + /// + /// Used by Bitcoin (Esplora) and Tron (`TronGrid`), whose APIs are REST + /// rather than JSON-RPC. `path` is relative to whatever base the host has + /// configured for `network`, without a leading slash. + /// + /// # Errors + /// + /// As [`Transport::json_rpc`]. A non-2xx status is + /// [`TransportError::Rpc`] when the body carries the API's own error and + /// [`TransportError::Unreachable`] when it does not. + async fn rest_get(&self, network: NetworkId, path: &str) -> TransportResult; + + /// Perform a REST POST and return the raw body. + /// + /// `path` is relative to the host's configured base for `network`, without + /// a leading slash. `content_type` is passed because these APIs are not + /// uniform: Esplora takes a raw transaction as `text/plain`, while + /// `TronGrid` expects `application/json`. + /// + /// # Errors + /// + /// As [`Transport::json_rpc`]. + async fn rest_post( + &self, + network: NetworkId, + path: &str, + body: String, + content_type: &str, + ) -> TransportResult; +} + +/// Deserialize a JSON-RPC `result` into a typed value. +/// +/// A small helper so every chain module does not repeat the same +/// `serde_json::from_value` plus error-mapping dance. A body that does not +/// match the expected shape is [`TransportError::Rpc`], not `Unreachable`: +/// the endpoint answered, it simply did not answer what was asked, and +/// retrying elsewhere will not fix a schema mismatch. +/// +/// # Errors +/// +/// [`TransportError::Rpc`] if `value` does not deserialize into `T`. +pub fn decode( + network: NetworkId, + method: &str, + value: Value, +) -> TransportResult { + serde_json::from_value(value).map_err(|e| TransportError::Rpc { + network, + message: format!("{method}: unexpected response shape: {e}"), + }) +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/rpc/test.rs b/src/openhuman/web3/wallet/primitives/rpc/test.rs new file mode 100644 index 0000000000..bd9fd5066d --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/rpc/test.rs @@ -0,0 +1,208 @@ +//! Unit tests for the transport seam. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use async_trait::async_trait; +use serde_json::{json, Value}; + +use super::{decode, NetworkId, Transport, TransportError, TransportResult}; +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// A transport that records what it was asked for and replays canned answers. +/// Stands in for a host implementation so the seam can be exercised without a +/// network. +struct FakeTransport { + answer: TransportResult, + calls: std::sync::Mutex>, +} + +impl FakeTransport { + fn ok(value: Value) -> Self { + Self { + answer: Ok(value), + calls: std::sync::Mutex::new(Vec::new()), + } + } + + fn err(error: TransportError) -> Self { + Self { + answer: Err(error), + calls: std::sync::Mutex::new(Vec::new()), + } + } + + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } +} + +#[async_trait] +impl Transport for FakeTransport { + async fn json_rpc( + &self, + _network: NetworkId, + method: &str, + params: Value, + ) -> TransportResult { + self.calls + .lock() + .unwrap() + .push(format!("json_rpc {method} {params}")); + self.answer.clone() + } + + async fn rest_get(&self, _network: NetworkId, path: &str) -> TransportResult { + self.calls.lock().unwrap().push(format!("rest_get {path}")); + self.answer.clone().map(|v| v.to_string()) + } + + async fn rest_post( + &self, + _network: NetworkId, + path: &str, + body: String, + content_type: &str, + ) -> TransportResult { + self.calls + .lock() + .unwrap() + .push(format!("rest_post {path} {content_type} {body}")); + self.answer.clone().map(|v| v.to_string()) + } +} + +#[test] +fn network_id_names_a_non_evm_chain_by_chain_alone() { + let id = NetworkId::chain(Chain::Solana); + assert_eq!(id.chain, Chain::Solana); + assert_eq!(id.evm_chain_id, None); + assert_eq!(id.to_string(), "solana"); +} + +#[test] +fn network_id_distinguishes_evm_networks_by_chain_id() { + // Ethereum and Base share an address format and an RPC dialect, so the + // chain alone cannot say which endpoint should answer. + let mainnet = NetworkId::evm(1); + let base = NetworkId::evm(8453); + assert_ne!(mainnet, base); + assert_eq!(mainnet.chain, base.chain); + assert_eq!(mainnet.to_string(), "evm:1"); + assert_eq!(base.to_string(), "evm:8453"); +} + +#[test] +fn unreachable_is_retryable_and_rpc_is_not() { + // The whole reason these are separate variants: a failover loop advances + // on the first and must stop dead on the second. + let network = NetworkId::chain(Chain::Btc); + let unreachable = TransportError::Unreachable { + network, + message: "connection refused".to_string(), + }; + let authoritative = TransportError::Rpc { + network, + message: "insufficient funds".to_string(), + }; + assert!(unreachable.is_retryable()); + assert!(!authoritative.is_retryable()); +} + +#[test] +fn errors_report_the_network_they_relate_to() { + let network = NetworkId::evm(8453); + let err = TransportError::Rpc { + network, + message: "nonce too low".to_string(), + }; + assert_eq!(err.network(), network); + assert!(err.to_string().contains("evm:8453")); + assert!(err.to_string().contains("nonce too low")); +} + +#[tokio::test] +async fn json_rpc_passes_the_method_and_params_through() { + let transport = FakeTransport::ok(json!("0x1")); + let out = transport + .json_rpc( + NetworkId::evm(1), + "eth_getTransactionCount", + json!(["0xabc", "latest"]), + ) + .await + .unwrap(); + assert_eq!(out, json!("0x1")); + assert_eq!( + transport.calls(), + vec![r#"json_rpc eth_getTransactionCount ["0xabc","latest"]"#.to_string()] + ); +} + +#[tokio::test] +async fn rest_post_carries_the_content_type() { + // Esplora wants text/plain for a raw transaction and TronGrid wants JSON, + // so the content type cannot be assumed by the caller. + let transport = FakeTransport::ok(json!("txid")); + transport + .rest_post( + NetworkId::chain(Chain::Btc), + "tx", + "0200000001".to_string(), + "text/plain", + ) + .await + .unwrap(); + assert_eq!( + transport.calls(), + vec!["rest_post tx text/plain 0200000001".to_string()] + ); +} + +#[tokio::test] +async fn a_transport_error_surfaces_to_the_caller_unchanged() { + let network = NetworkId::chain(Chain::Solana); + let transport = FakeTransport::err(TransportError::Rpc { + network, + message: "blockhash not found".to_string(), + }); + let err = transport + .json_rpc(network, "sendTransaction", json!([])) + .await + .unwrap_err(); + assert!(!err.is_retryable()); + assert!(err.to_string().contains("blockhash not found")); +} + +#[test] +fn decode_turns_a_matching_result_into_a_typed_value() { + #[derive(serde::Deserialize, PartialEq, Debug)] + struct Balance { + value: u64, + } + let out: Balance = decode( + NetworkId::chain(Chain::Solana), + "getBalance", + json!({"value": 42}), + ) + .unwrap(); + assert_eq!(out, Balance { value: 42 }); +} + +#[test] +fn decode_reports_a_shape_mismatch_as_authoritative_not_retryable() { + // The endpoint answered; it just did not answer what was asked. Retrying + // elsewhere cannot fix a schema mismatch, so this must not be Unreachable. + #[derive(serde::Deserialize, Debug)] + struct Balance { + #[allow(dead_code)] + value: u64, + } + let err = decode::( + NetworkId::chain(Chain::Solana), + "getBalance", + json!({"nope": true}), + ) + .unwrap_err(); + assert!(!err.is_retryable(), "a shape mismatch is not retryable"); + assert!(err.to_string().contains("getBalance")); +} diff --git a/src/openhuman/web3/wallet/primitives/wire/mod.rs b/src/openhuman/web3/wallet/primitives/wire/mod.rs new file mode 100644 index 0000000000..338e7df654 --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/wire/mod.rs @@ -0,0 +1,279 @@ +//! The wire contract between a host and a signing backend. +//! +//! # Why this module exists, and why it has no dependencies +//! +//! A host can run this crate's transaction building in-process, or it can run +//! it somewhere else — most usefully in a loadable module, so the chain +//! libraries that building requires (`bitcoin` and its native `secp256k1` +//! build, above all) are absent from the host binary entirely. +//! +//! For that second arrangement both sides must agree on a set of types, and +//! **only the host side may be free of the heavy dependencies**. So these types +//! live outside every format gate and pull in nothing but `serde`: a host can +//! take this crate with `default-features = false`, get the whole contract, and +//! still not link a single chain library. It is the same carve-out +//! `crate::openhuman::tools::implementations::document::format::spec` makes for documents. +//! +//! # The split: building is not signing +//! +//! Every type here exists to serve one rule — **key material never crosses this +//! boundary**. A backend receives transaction fields and returns the bytes that +//! need signing; the host signs them; the backend reassembles. Two round trips +//! instead of one, in exchange for a private key that never leaves the process +//! that owns it. +//! +//! That constraint is what shapes the API. A [`SigningRequest`] carries no +//! secret, and an [`AttachRequest`] carries the original fields **again** +//! alongside the signatures, rather than a handle to something the backend +//! remembered. A backend holding half-built transactions between calls would +//! need a store, bounds on that store, and an expiry policy for callers that +//! never come back — all of which is avoided by rebuilding. Building is +//! deterministic, so rebuilding from the same fields yields the same +//! transaction the digests were computed over. +//! +//! # Signature shapes +//! +//! Three of the four chains sign a 32-byte digest with secp256k1 ECDSA and need +//! the recovery id; Solana signs the message itself with ed25519 and does not. +//! [`Signature`] is an enum over exactly those two cases rather than a bag of +//! bytes, so a host cannot hand back an ed25519 signature for an EVM +//! transaction and have it fail somewhere deep in reassembly. + +use serde::{Deserialize, Serialize}; + +use crate::openhuman::web3::wallet::primitives::chain::Chain; + +/// Bytes a host must sign, and how. +/// +/// For secp256k1 chains this is a 32-byte digest that is signed directly — +/// **already hashed**, so a host must use a "sign prehash" entry point and must +/// not hash it again. For Solana it is the full serialized message, because +/// ed25519 hashes internally as part of signing. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SigningPayload { + /// Lowercase hex of the bytes to sign. + pub bytes_hex: String, + /// Which signing scheme these bytes expect. + pub scheme: Scheme, +} + +/// How a [`SigningPayload`] must be signed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[non_exhaustive] +pub enum Scheme { + /// secp256k1 ECDSA over an already-computed 32-byte digest, low-`s` + /// normalized, with the recovery id retained. + /// + /// Low-`s` is not optional: Bitcoin enforces it as a relay policy rule + /// (BIP-146) and Ethereum as a consensus rule (EIP-2), so a high-`s` + /// signature produces a transaction that is rejected rather than one that + /// merely looks different. Both `k256` and `secp256k1` normalize by + /// default; a host that implements signing itself must not skip it. + Secp256k1Prehash, + /// ed25519 over the full message, which the scheme hashes itself. + Ed25519, +} + +/// A signature handed back to a backend for reassembly. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "scheme", rename_all = "snake_case")] +#[non_exhaustive] +pub enum Signature { + /// secp256k1 ECDSA: 32-byte `r`, 32-byte `s`, and the recovery id. + Secp256k1 { + /// Lowercase hex of `r || s`, exactly 64 bytes. + rs_hex: String, + /// Recovery id, 0..=3. + /// + /// Carried even for Bitcoin, which does not use it, so one variant + /// serves all three secp256k1 chains. EVM folds it into EIP-155 `v` + /// and Tron appends it directly. + recovery_id: u8, + }, + /// ed25519: the 64-byte signature. + Ed25519 { + /// Lowercase hex of the signature, exactly 64 bytes. + signature_hex: String, + }, +} + +/// The public key controlling the account a transaction spends from. +/// +/// Public by definition, so unlike the secret it may cross the boundary freely. +/// A backend needs it for two things: Bitcoin puts it in the witness, and every +/// chain uses it to check that the key the host is about to sign with actually +/// controls the `from` address — a mismatch that would otherwise surface as an +/// unspendable broadcast transaction. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicKey { + /// Lowercase hex. Compressed SEC1 (33 bytes) for secp256k1 chains, the + /// 32-byte public key for ed25519. + pub key_hex: String, +} + +/// Ask a backend what needs signing for a transaction. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SigningRequest { + /// The transaction to build, which names its own chain. + /// + /// There is deliberately no separate `chain` field. Carrying one alongside + /// this would let a request say `btc` while holding an EVM transaction — + /// a state the backend would have to detect and reject at runtime. Reading + /// the chain off the variant instead makes that disagreement unrepresentable. + pub transaction: TransactionSpec, + /// The public key that will sign. + pub public_key: PublicKey, +} + +/// Hand signatures back so a backend can assemble the final transaction. +/// +/// Carries `transaction` again rather than a handle: see the module docs on why +/// a backend deliberately keeps no state between the two calls. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AttachRequest { + /// The same fields passed to the matching [`SigningRequest`]. + /// + /// As there, the chain comes from the variant rather than a parallel field. + pub transaction: TransactionSpec, + /// The public key that signed. + pub public_key: PublicKey, + /// One signature per [`SigningPayload`] returned, in the same order. + /// + /// Bitcoin needs one per selected input; the other three need exactly one. + pub signatures: Vec, +} + +/// What a backend answers a [`SigningRequest`] with. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UnsignedTransaction { + /// Everything that needs a signature, in the order the signatures must be + /// returned. + pub payloads: Vec, +} + +/// What a backend answers an [`AttachRequest`] with. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SignedTransaction { + /// The broadcast-ready transaction, in whatever encoding the chain's RPC + /// expects: hex for Bitcoin, EVM and Tron, base64 for Solana. + pub raw: String, + /// The transaction id or hash a node will report, when the chain lets it be + /// computed locally. + pub txid: Option, +} + +/// A transaction to build, per chain. +/// +/// One enum rather than four methods so a host holds a single value and the +/// chain tag cannot disagree with the fields — the mismatch a pair of parallel +/// arguments would allow. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +#[non_exhaustive] +pub enum TransactionSpec { + /// A Bitcoin P2WPKH spend. + Btc { + /// Sender address; must be P2WPKH. + from: String, + /// Recipient address; any mainnet type. + to: String, + /// Amount in satoshis. + amount_sat: u64, + /// Absolute fee in satoshis. + /// + /// Bitcoin's fee is implicit — `sum(inputs) - sum(outputs)` — so it is + /// stated here rather than derived from a rate. A caller that thinks + /// in sat/vB converts before sending. + fee_sat: u64, + /// Every spendable output held by `from`. + utxos: Vec, + }, + /// An EVM legacy transaction. + Evm { + /// Recipient — the token contract for an ERC-20 transfer. + to: String, + /// Value in wei. + value_wei: String, + /// Call data, `0x`-prefixed hex. Empty for a native transfer. + data_hex: String, + /// Sender nonce. + nonce: u64, + /// Gas limit. + gas_limit: u64, + /// Gas price in wei. + gas_price_wei: String, + /// EIP-155 chain id. + chain_id: u64, + }, + /// A Solana native SOL transfer. + Solana { + /// Sender address. + from: String, + /// Recipient address. + to: String, + /// Amount in lamports. + lamports: u64, + /// A recent blockhash, base58. + recent_blockhash: String, + }, + /// A Tron transfer, already assembled by the node. + /// + /// Tron is the odd one out: `createtransaction` builds the transaction + /// server-side and returns it, so there is nothing for this crate to build + /// — only a payload to verify and sign. The verification is the point, and + /// it is why the recipient and amount are carried alongside: a node that + /// returned a transaction paying somebody else would otherwise be signed + /// without complaint. + Tron { + /// The node's `raw_data_hex`. + raw_data_hex: String, + /// The recipient the caller intended, base58check. + expected_to: String, + /// The txid the node reported, to be recomputed and compared. + expected_txid: String, + }, +} + +impl TransactionSpec { + /// Which chain this transaction belongs to. + /// + /// The single source of truth for the chain, which is why neither request + /// type carries it separately. + /// + /// Infallible, and deliberately so despite `#[non_exhaustive]`. That + /// attribute binds only *downstream* crates, and a downstream crate calls + /// this method rather than matching the enum itself — so there is no + /// wildcard arm to write here, and adding a variant is a compile error in + /// this file, which is where it should be caught. + #[must_use] + pub fn chain(&self) -> Chain { + match self { + Self::Btc { .. } => Chain::Btc, + Self::Evm { .. } => Chain::Evm, + Self::Solana { .. } => Chain::Solana, + Self::Tron { .. } => Chain::Tron, + } + } +} + +/// One spendable Bitcoin output. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Utxo { + /// Transaction id holding this output. + pub txid: String, + /// Output index within that transaction. + pub vout: u32, + /// Value in satoshis. + pub value: u64, +} + +#[cfg(test)] +mod test; diff --git a/src/openhuman/web3/wallet/primitives/wire/test.rs b/src/openhuman/web3/wallet/primitives/wire/test.rs new file mode 100644 index 0000000000..8e87de3eec --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/wire/test.rs @@ -0,0 +1,238 @@ +//! Tests for the host/backend wire contract. +//! +//! These are contract tests, not logic tests: the module holds no behaviour. +//! What can break here is compatibility — a field renamed, a tag changed, an +//! enum representation altered — and each of those breaks a host and a backend +//! that were built from different revisions, at runtime, with a deserialization +//! error rather than a compile failure. So the shapes are pinned literally. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use serde_json::json; + +use super::{ + AttachRequest, PublicKey, Scheme, Signature, SignedTransaction, SigningPayload, SigningRequest, + TransactionSpec, UnsignedTransaction, Utxo, +}; + +#[test] +fn a_signing_request_round_trips_through_json() { + let request = SigningRequest { + transaction: TransactionSpec::Evm { + to: "0x1111111111111111111111111111111111111111".to_string(), + value_wei: "1000".to_string(), + data_hex: "0x".to_string(), + nonce: 7, + gas_limit: 21_000, + gas_price_wei: "20000000000".to_string(), + chain_id: 1, + }, + public_key: PublicKey { + key_hex: "02".repeat(33), + }, + }; + + let encoded = serde_json::to_string(&request).unwrap(); + let decoded: SigningRequest = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded, request); +} + +#[test] +fn the_transaction_spec_tag_is_the_published_one() { + // A host and a backend from different revisions meet here. The tag and the + // field names are the contract, so they are asserted against literals + // rather than against a re-serialization of the same value, which would + // agree with itself no matter what it was renamed to. + let spec = TransactionSpec::Solana { + from: "11111111111111111111111111111112".to_string(), + to: "11111111111111111111111111111113".to_string(), + lamports: 5, + recent_blockhash: "11111111111111111111111111111114".to_string(), + }; + assert_eq!( + serde_json::to_value(&spec).unwrap(), + json!({ + "kind": "solana", + "from": "11111111111111111111111111111112", + "to": "11111111111111111111111111111113", + "lamports": 5, + "recent_blockhash": "11111111111111111111111111111114", + }) + ); +} + +#[test] +fn a_signature_is_tagged_by_its_scheme() { + assert_eq!( + serde_json::to_value(Signature::Secp256k1 { + rs_hex: "ab".repeat(64), + recovery_id: 1, + }) + .unwrap(), + json!({ "scheme": "secp256k1", "rs_hex": "ab".repeat(64), "recovery_id": 1 }) + ); + assert_eq!( + serde_json::to_value(Signature::Ed25519 { + signature_hex: "cd".repeat(64), + }) + .unwrap(), + json!({ "scheme": "ed25519", "signature_hex": "cd".repeat(64) }) + ); +} + +#[test] +fn an_ed25519_signature_cannot_deserialize_as_a_secp256k1_one() { + // The enum is tagged precisely so a host cannot return the wrong scheme's + // signature and have it fail deep inside reassembly instead of at the + // boundary. + let ed = json!({ "scheme": "ed25519", "signature_hex": "cd".repeat(64) }); + let decoded: Signature = serde_json::from_value(ed).unwrap(); + assert!(matches!(decoded, Signature::Ed25519 { .. })); + + let mismatched = json!({ + "scheme": "secp256k1", + "signature_hex": "cd".repeat(64) + }); + assert!(serde_json::from_value::(mismatched).is_err()); +} + +#[test] +fn unknown_fields_are_refused_rather_than_ignored() { + // A backend newer than its host would otherwise silently drop a field it + // was told about, which for a transaction means signing something other + // than what was asked for. + let with_extra = json!({ + "txid": "aa".repeat(32), + "vout": 0, + "value": 1000, + "surprise": true, + }); + assert!(serde_json::from_value::(with_extra).is_err()); +} + +#[test] +fn the_signing_scheme_names_are_stable() { + assert_eq!( + serde_json::to_value(Scheme::Secp256k1Prehash).unwrap(), + json!("secp256k1_prehash") + ); + assert_eq!( + serde_json::to_value(Scheme::Ed25519).unwrap(), + json!("ed25519") + ); +} + +#[test] +fn an_attach_request_carries_one_signature_per_payload() { + // Not a rule the type can enforce, but the pairing is the contract: the + // Bitcoin path returns one payload per selected input and expects them + // back in the same order. + let unsigned = UnsignedTransaction { + payloads: vec![ + SigningPayload { + bytes_hex: "11".repeat(32), + scheme: Scheme::Secp256k1Prehash, + }, + SigningPayload { + bytes_hex: "22".repeat(32), + scheme: Scheme::Secp256k1Prehash, + }, + ], + }; + let attach = AttachRequest { + transaction: TransactionSpec::Btc { + from: "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4".to_string(), + to: "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4".to_string(), + amount_sat: 1_000, + fee_sat: 5, + utxos: vec![], + }, + public_key: PublicKey { + key_hex: "02".repeat(33), + }, + signatures: vec![ + Signature::Secp256k1 { + rs_hex: "ab".repeat(64), + recovery_id: 0, + }, + Signature::Secp256k1 { + rs_hex: "cd".repeat(64), + recovery_id: 1, + }, + ], + }; + assert_eq!(attach.signatures.len(), unsigned.payloads.len()); + + let encoded = serde_json::to_string(&attach).unwrap(); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + attach + ); +} + +#[test] +fn a_signed_transaction_may_omit_a_locally_unknowable_txid() { + let signed = SignedTransaction { + raw: "0xdeadbeef".to_string(), + txid: None, + }; + let encoded = serde_json::to_string(&signed).unwrap(); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + signed + ); +} + +#[test] +fn every_transaction_names_its_own_chain() { + // `chain()` is the single source of truth now that the requests carry no + // `chain` field, so a wrong arm here would route a transaction to the + // wrong chain's builder — with a real key already loaded. + use crate::openhuman::web3::wallet::primitives::chain::Chain; + + let cases = [ + ( + TransactionSpec::Btc { + from: String::new(), + to: String::new(), + amount_sat: 0, + fee_sat: 0, + utxos: Vec::new(), + }, + Chain::Btc, + ), + ( + TransactionSpec::Evm { + to: String::new(), + value_wei: "0".to_string(), + data_hex: String::new(), + nonce: 0, + gas_limit: 0, + gas_price_wei: "0".to_string(), + chain_id: 1, + }, + Chain::Evm, + ), + ( + TransactionSpec::Solana { + from: String::new(), + to: String::new(), + lamports: 0, + recent_blockhash: String::new(), + }, + Chain::Solana, + ), + ( + TransactionSpec::Tron { + raw_data_hex: String::new(), + expected_to: String::new(), + expected_txid: String::new(), + }, + Chain::Tron, + ), + ]; + + for (spec, expected) in cases { + assert_eq!(spec.chain(), expected); + } +} diff --git a/src/openhuman/web3/wallet/primitives/x402/mod.rs b/src/openhuman/web3/wallet/primitives/x402/mod.rs new file mode 100644 index 0000000000..e0f949106d --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/x402/mod.rs @@ -0,0 +1,45 @@ +//! The x402 machine-payment protocol (v2). +//! +//! x402 revives HTTP's long-unused `402 Payment Required`. A server answers a +//! request with a 402 and a `PAYMENT-REQUIRED` header describing what it will +//! accept; the client pays, retries with a `PAYMENT-SIGNATURE` header carrying +//! the proof, and the server settles it through a facilitator and answers with +//! `PAYMENT-RESPONSE`. +//! +//! This module owns the **wire types** — the header payloads and the rules for +//! reading them. Every header payload is standard-base64-encoded JSON, and +//! networks are named in [CAIP-2] form (`solana:…`, `eip155:8453`). +//! +//! ## Amounts are strings, and that is not laziness +//! +//! [`PaymentRequirements::amount`] is a `String` of atomic units, not a number. +//! JSON numbers are IEEE 754 doubles in most parsers, which cannot represent +//! every `u64` exactly — and a token amount that survives a round trip through +//! a JavaScript facilitator only approximately is a payment for the wrong sum. +//! The protocol carries them as decimal strings for that reason, and so does +//! this module. +//! +//! ## The client signs an authorisation; the facilitator broadcasts +//! +//! In both supported schemes the payer never broadcasts. On Solana it hands +//! over a partially-signed transaction that the facilitator co-signs as fee +//! payer; on EVM it signs an EIP-3009 `transferWithAuthorization` the +//! facilitator submits. So a payment proof is a *capability someone else will +//! exercise* — which is why [`EvmAuthorization`] carries `valid_after`, +//! `valid_before` and a `nonce`: without them an authorisation would be +//! replayable indefinitely. +//! +//! [CAIP-2]: https://chainagnostic.org/CAIPs/caip-2 + +mod types; + +#[allow(unused_imports)] +pub use types::{ + EvmAuthorization, EvmPaymentProof, PaymentChain, PaymentExtra, PaymentPayload, PaymentProof, + PaymentRequired, PaymentRequirements, ResourceInfo, SettlementResponse, SolanaPaymentProof, + BASE_MAINNET_CAIP2, BASE_SEPOLIA_CAIP2, COMPUTE_BUDGET_PROGRAM, ETHEREUM_MAINNET_CAIP2, + HEADER_PAYMENT_REQUIRED, HEADER_PAYMENT_REQUIRED_V1, HEADER_PAYMENT_RESPONSE, + HEADER_PAYMENT_SIGNATURE, HEADER_PAYMENT_SIGNATURE_V1, SOLANA_DEVNET_CAIP2, + SOLANA_MAINNET_CAIP2, SPL_MEMO_PROGRAM, SPL_TOKEN_PROGRAM, USDC_BASE_MAINNET, + USDC_BASE_SEPOLIA, USDC_ETHEREUM_MAINNET, USDC_MINT_DEVNET, USDC_MINT_MAINNET, X402_VERSION, +}; diff --git a/src/openhuman/web3/wallet/primitives/x402/types.rs b/src/openhuman/web3/wallet/primitives/x402/types.rs new file mode 100644 index 0000000000..a69ac18c0b --- /dev/null +++ b/src/openhuman/web3/wallet/primitives/x402/types.rs @@ -0,0 +1,513 @@ +//! Wire types for the x402 protocol (v2). +//! +//! All header payloads are standard-base64-encoded JSON. Network identifiers +//! use CAIP-2 format (e.g. `solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp`). + +use serde::{Deserialize, Serialize}; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/// The protocol version this module implements. +pub const X402_VERSION: u8 = 2; + +/// Response header carrying the v2 402 challenge. +pub const HEADER_PAYMENT_REQUIRED: &str = "PAYMENT-REQUIRED"; +/// The v1 spelling of the challenge header, still sent by some servers. +pub const HEADER_PAYMENT_REQUIRED_V1: &str = "X-PAYMENT-REQUIRED"; +/// Request header carrying the v2 payment proof. +pub const HEADER_PAYMENT_SIGNATURE: &str = "PAYMENT-SIGNATURE"; +/// The v1 spelling of the payment-proof header. +pub const HEADER_PAYMENT_SIGNATURE_V1: &str = "X-PAYMENT"; +/// Response header carrying the settlement result. +pub const HEADER_PAYMENT_RESPONSE: &str = "PAYMENT-RESPONSE"; + +/// CAIP-2 identifier for Solana mainnet-beta. +pub const SOLANA_MAINNET_CAIP2: &str = "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"; +/// CAIP-2 identifier for Solana devnet. +pub const SOLANA_DEVNET_CAIP2: &str = "solana:EtWTRABZaYq6iMfeYKouRu166VU2xqa1"; + +/// USDC SPL mint on Solana mainnet-beta. +pub const USDC_MINT_MAINNET: &str = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v"; +/// USDC SPL mint on Solana devnet. Differs from mainnet. +pub const USDC_MINT_DEVNET: &str = "4zMMC9srt5Ri5X14GAgXhaHii3GnPAEERYPJgZJDncDU"; + +/// The SPL Token program id. +pub const SPL_TOKEN_PROGRAM: &str = "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"; +/// The SPL Memo program id, used for payment uniqueness. +pub const SPL_MEMO_PROGRAM: &str = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"; +/// The Compute Budget program id. +pub const COMPUTE_BUDGET_PROGRAM: &str = "ComputeBudget111111111111111111111111111111"; + +// EVM / Base chain constants (CAIP-2 format: eip155:) +/// CAIP-2 identifier for Base mainnet. +pub const BASE_MAINNET_CAIP2: &str = "eip155:8453"; +/// CAIP-2 identifier for Base Sepolia. +pub const BASE_SEPOLIA_CAIP2: &str = "eip155:84532"; +/// CAIP-2 identifier for Ethereum mainnet. +pub const ETHEREUM_MAINNET_CAIP2: &str = "eip155:1"; + +/// USDC contract on Base mainnet. +pub const USDC_BASE_MAINNET: &str = "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913"; +/// USDC contract on Base Sepolia. +pub const USDC_BASE_SEPOLIA: &str = "0x036CbD53842c5426634e7929541eC2318f3dCF7e"; +/// USDC contract on Ethereum mainnet. +pub const USDC_ETHEREUM_MAINNET: &str = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"; + +// --------------------------------------------------------------------------- +// 402 challenge — server → client (PAYMENT-REQUIRED header) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +/// The 402 challenge a server sends: what it will accept, and for what. +pub struct PaymentRequired { + /// See the x402 v2 specification. + pub x402_version: u8, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub error: Option, + /// See the x402 v2 specification. + pub resource: ResourceInfo, + /// See the x402 v2 specification. + pub accepts: Vec, + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + /// See the x402 v2 specification. + pub extensions: serde_json::Map, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +/// The resource a payment buys access to. +pub struct ResourceInfo { + /// See the x402 v2 specification. + pub url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub mime_type: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +/// One payment option a server will accept. +pub struct PaymentRequirements { + /// See the x402 v2 specification. + pub scheme: String, + /// See the x402 v2 specification. + pub network: String, + /// Amount in atomic token units, as a decimal string (1 USDC = `1000000`). + /// + /// A string rather than a number — see the module docs. + /// See the x402 v2 specification. + pub amount: String, + /// Token mint address (Solana) or contract address (EVM). + /// See the x402 v2 specification. + pub asset: String, + /// Recipient wallet address. + /// See the x402 v2 specification. + pub pay_to: String, + /// See the x402 v2 specification. + pub max_timeout_seconds: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub extra: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +/// Scheme-specific extras a server attaches to a requirement. +pub struct PaymentExtra { + /// Facilitator pubkey that will co-sign as fee payer (Solana). + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub fee_payer: Option, + /// Required memo value for transaction uniqueness (Solana). + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub memo: Option, + /// EIP-712 domain name for the token contract (EVM, e.g. "USD Coin"). + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub name: Option, + /// EIP-712 domain version for the token contract (EVM, e.g. "2"). + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub version: Option, +} + +// --------------------------------------------------------------------------- +// Payment proof — client → server (PAYMENT-SIGNATURE header) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +/// The proof a client sends back after paying. +pub struct PaymentPayload { + /// See the x402 v2 specification. + pub x402_version: u8, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub resource: Option, + /// See the x402 v2 specification. + pub accepted: PaymentRequirements, + /// See the x402 v2 specification. + pub payload: PaymentProof, + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + /// See the x402 v2 specification. + pub extensions: serde_json::Map, +} + +/// Chain-specific payment proof. Serializes flat (untagged) so the facilitator +/// sees either `{ "transaction": "..." }` (Solana) or +/// `{ "signature": "0x...", "authorization": {...} }` (EVM). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +/// A chain-specific payment proof. +/// +/// Serialises untagged, so a facilitator sees the chain's object directly. +pub enum PaymentProof { + /// A Solana partially-signed transaction. + Solana(SolanaPaymentProof), + /// An EVM EIP-3009 authorisation. + Evm(EvmPaymentProof), +} + +/// Solana `exact` scheme payload — a partially-signed `VersionedTransaction` +/// serialized as standard base64. The facilitator adds its fee-payer signature +/// and broadcasts. +#[derive(Debug, Clone, Serialize, Deserialize)] +/// Solana `exact` proof: a partially-signed transaction, base64. +/// +/// The facilitator adds its fee-payer signature and broadcasts. +pub struct SolanaPaymentProof { + /// See the x402 v2 specification. + pub transaction: String, +} + +/// EVM `exact` scheme payload — a signed EIP-3009 `transferWithAuthorization` +/// or plain ERC-20 transfer authorization for the facilitator to submit. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +/// EVM `exact` proof: a signed EIP-3009 authorisation for the +/// facilitator to submit. +pub struct EvmPaymentProof { + /// See the x402 v2 specification. + pub signature: String, + /// See the x402 v2 specification. + pub authorization: EvmAuthorization, +} + +/// EIP-3009 `transferWithAuthorization` parameters signed by the token holder. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +/// EIP-3009 `transferWithAuthorization` parameters signed by the token +/// holder. +/// +/// `valid_after`, `valid_before` and `nonce` are what stop the +/// authorisation being replayable — see the module docs. +pub struct EvmAuthorization { + /// See the x402 v2 specification. + pub from: String, + /// See the x402 v2 specification. + pub to: String, + /// See the x402 v2 specification. + pub value: String, + /// See the x402 v2 specification. + pub valid_after: String, + /// See the x402 v2 specification. + pub valid_before: String, + /// See the x402 v2 specification. + pub nonce: String, +} + +// --------------------------------------------------------------------------- +// Settlement response — server → client (PAYMENT-RESPONSE header) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +/// The settlement result a server returns once the payment landed. +pub struct SettlementResponse { + /// See the x402 v2 specification. + pub success: bool, + /// Base58 transaction signature (Solana) or hex tx hash (EVM). + /// See the x402 v2 specification. + pub transaction: String, + /// See the x402 v2 specification. + pub network: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub payer: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub error_reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + /// See the x402 v2 specification. + pub amount: Option, + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + /// See the x402 v2 specification. + pub extensions: serde_json::Map, +} + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +impl PaymentRequired { + /// Find the first `accepts` entry whose network starts with `"solana:"` and + /// whose scheme is `"exact"`. + #[must_use] + pub fn solana_exact_requirement(&self) -> Option<&PaymentRequirements> { + self.accepts + .iter() + .find(|r| r.scheme == "exact" && r.network.starts_with("solana:")) + } + + /// Find the first `accepts` entry whose network starts with `"eip155:"` and + /// whose scheme is `"exact"`. + #[must_use] + pub fn evm_exact_requirement(&self) -> Option<&PaymentRequirements> { + self.accepts + .iter() + .find(|r| r.scheme == "exact" && r.network.starts_with("eip155:")) + } + + /// The preferred payment option: **Solana first, then EVM**. + /// + /// The order matters to a payer with funds on both chains, so it is stated + /// plainly here. The implementation this was extracted from carried a doc + /// comment claiming the opposite ("prefer EVM (Base), fall back to + /// Solana") while the code checked Solana first; the code's behaviour is + /// preserved and the comment corrected, since changing which chain a payer + /// spends from is not a documentation fix. + #[must_use] + pub fn best_exact_requirement(&self) -> Option<(&PaymentRequirements, PaymentChain)> { + if let Some(sol) = self.solana_exact_requirement() { + Some((sol, PaymentChain::Solana)) + } else if let Some(evm) = self.evm_exact_requirement() { + Some((evm, PaymentChain::Evm)) + } else { + None + } + } +} + +/// Which chain family a payment requirement targets. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Which chain family a payment requirement targets. +pub enum PaymentChain { + /// A Solana `exact`-scheme payment. + Solana, + /// An EVM `exact`-scheme payment. + Evm, +} + +impl PaymentRequirements { + /// Whether this requirement targets Solana mainnet-beta. + #[must_use] + pub fn is_solana_mainnet(&self) -> bool { + self.network == SOLANA_MAINNET_CAIP2 + } + + /// Whether this requirement targets Base mainnet. + #[must_use] + pub fn is_base_mainnet(&self) -> bool { + self.network == BASE_MAINNET_CAIP2 + } + + /// Parse the EVM chain ID from an `eip155:` network string. + #[must_use] + pub fn evm_chain_id(&self) -> Option { + self.network + .strip_prefix("eip155:") + .and_then(|s| s.parse().ok()) + } + + /// The facilitator pubkey that will co-sign as fee payer, if the server + /// named one. + #[must_use] + pub fn fee_payer_pubkey(&self) -> Option<&str> { + self.extra.as_ref()?.fee_payer.as_deref() + } + + /// The memo the server requires for transaction uniqueness, if any. + #[must_use] + pub fn memo_value(&self) -> Option<&str> { + self.extra.as_ref()?.memo.as_deref() + } +} + +#[cfg(test)] +mod test { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::{ + PaymentChain, PaymentRequired, PaymentRequirements, BASE_MAINNET_CAIP2, + SOLANA_MAINNET_CAIP2, X402_VERSION, + }; + + fn requirement(scheme: &str, network: &str) -> PaymentRequirements { + PaymentRequirements { + scheme: scheme.to_string(), + network: network.to_string(), + amount: "1000000".to_string(), + asset: super::USDC_MINT_MAINNET.to_string(), + pay_to: "11111111111111111111111111111111".to_string(), + max_timeout_seconds: 60, + extra: None, + } + } + + fn challenge(accepts: Vec) -> PaymentRequired { + PaymentRequired { + x402_version: X402_VERSION, + error: None, + resource: super::ResourceInfo { + url: "https://example.test/thing".to_string(), + description: None, + mime_type: None, + }, + accepts, + extensions: serde_json::Map::new(), + } + } + + #[test] + fn only_the_exact_scheme_is_selected() { + // A server may offer schemes this crate cannot pay; picking one of + // those would produce a proof the facilitator rejects. + let c = challenge(vec![requirement("upto", SOLANA_MAINNET_CAIP2)]); + assert!(c.solana_exact_requirement().is_none()); + assert!(c.best_exact_requirement().is_none()); + } + + #[test] + fn requirements_are_matched_by_network_prefix_not_exact_string() { + // CAIP-2 names a specific chain, so devnet and mainnet differ — but + // both are Solana, and the selector must accept either. + let c = challenge(vec![requirement("exact", super::SOLANA_DEVNET_CAIP2)]); + assert!(c.solana_exact_requirement().is_some()); + + let c = challenge(vec![requirement("exact", super::BASE_SEPOLIA_CAIP2)]); + assert!(c.evm_exact_requirement().is_some()); + } + + #[test] + fn solana_is_preferred_when_both_are_offered() { + // Pinning the documented order: which chain a payer spends from is + // observable behaviour, not an implementation detail. + let c = challenge(vec![ + requirement("exact", BASE_MAINNET_CAIP2), + requirement("exact", SOLANA_MAINNET_CAIP2), + ]); + let (_, chain) = c.best_exact_requirement().unwrap(); + assert_eq!(chain, PaymentChain::Solana); + } + + #[test] + fn evm_is_used_when_it_is_the_only_option() { + let c = challenge(vec![requirement("exact", BASE_MAINNET_CAIP2)]); + let (req, chain) = c.best_exact_requirement().unwrap(); + assert_eq!(chain, PaymentChain::Evm); + assert_eq!(req.evm_chain_id(), Some(8453)); + } + + #[test] + fn the_evm_chain_id_is_parsed_from_the_caip2_network() { + assert_eq!( + requirement("exact", "eip155:1").evm_chain_id(), + Some(1), + "ethereum mainnet" + ); + assert_eq!( + requirement("exact", SOLANA_MAINNET_CAIP2).evm_chain_id(), + None, + "a Solana network has no EVM chain id" + ); + assert_eq!( + requirement("exact", "eip155:notanumber").evm_chain_id(), + None + ); + } + + #[test] + fn amounts_stay_strings_through_a_json_round_trip() { + // The reason the protocol uses strings: a u64 amount through a + // double-based JSON parser can come back as a different number. + let mut req = requirement("exact", SOLANA_MAINNET_CAIP2); + req.amount = "18446744073709551615".to_string(); // u64::MAX + let json = serde_json::to_string(&req).unwrap(); + let back: PaymentRequirements = serde_json::from_str(&json).unwrap(); + assert_eq!(back.amount, "18446744073709551615"); + } + + #[test] + fn the_wire_shape_is_camel_case() { + // The header payload is read by facilitators in other languages, so + // the field names are part of the contract. + let json = serde_json::to_string(&requirement("exact", SOLANA_MAINNET_CAIP2)).unwrap(); + assert!(json.contains("\"payTo\""), "{json}"); + assert!(json.contains("\"maxTimeoutSeconds\""), "{json}"); + assert!(!json.contains("pay_to"), "{json}"); + } + + #[test] + fn a_payment_proof_serialises_untagged() { + // The facilitator sees the chain-specific object directly, with no + // enum discriminant wrapping it. + let solana = super::PaymentProof::Solana(super::SolanaPaymentProof { + transaction: "base64tx".to_string(), + }); + let json = serde_json::to_string(&solana).unwrap(); + assert_eq!(json, r#"{"transaction":"base64tx"}"#); + + let evm = super::PaymentProof::Evm(super::EvmPaymentProof { + signature: "0xsig".to_string(), + authorization: super::EvmAuthorization { + from: "0xa".to_string(), + to: "0xb".to_string(), + value: "1".to_string(), + valid_after: "0".to_string(), + valid_before: "99".to_string(), + nonce: "0xn".to_string(), + }, + }); + let json = serde_json::to_string(&evm).unwrap(); + assert!(json.starts_with(r#"{"signature":"0xsig""#), "{json}"); + assert!(json.contains("\"validBefore\""), "{json}"); + } + + #[test] + fn optional_fields_are_omitted_rather_than_sent_as_null() { + let json = serde_json::to_string(&requirement("exact", SOLANA_MAINNET_CAIP2)).unwrap(); + assert!(!json.contains("extra"), "absent extras are omitted: {json}"); + } + + #[test] + fn a_challenge_round_trips() { + let c = challenge(vec![requirement("exact", SOLANA_MAINNET_CAIP2)]); + let json = serde_json::to_string(&c).unwrap(); + let back: PaymentRequired = serde_json::from_str(&json).unwrap(); + assert_eq!(back.x402_version, X402_VERSION); + assert_eq!(back.accepts.len(), 1); + assert_eq!(back.resource.url, "https://example.test/thing"); + } + + #[test] + fn unknown_extension_fields_are_preserved_not_rejected() { + // Unlike the document spec, this is a protocol other implementations + // extend, so an unknown key must not fail the parse. + let json = r#"{ + "x402Version": 2, + "resource": { "url": "https://example.test" }, + "accepts": [], + "extensions": { "somethingNew": true } + }"#; + let parsed: PaymentRequired = serde_json::from_str(json).unwrap(); + assert!(parsed.extensions.contains_key("somethingNew")); + } +} diff --git a/src/openhuman/web3/wallet/transport.rs b/src/openhuman/web3/wallet/transport.rs index 3c930e71d5..e87d5fbeb9 100644 --- a/src/openhuman/web3/wallet/transport.rs +++ b/src/openhuman/web3/wallet/transport.rs @@ -1,8 +1,8 @@ -//! OpenHuman's implementation of the [`tinywallet::rpc::Transport`] seam. +//! OpenHuman's implementation of the [`crate::openhuman::web3::wallet::primitives::rpc::Transport`] seam. //! //! `tinywallet` performs no I/O and takes no URLs: it names a -//! [`NetworkId`](tinywallet::rpc::NetworkId) and asks a host to reach it. This -//! module is that host side — the adapter that lets `tinywallet::client` and +//! [`NetworkId`](crate::openhuman::web3::wallet::primitives::rpc::NetworkId) and asks a host to reach it. This +//! module is that host side — the adapter that lets `crate::openhuman::web3::wallet::primitives::client` and //! the chain modules run against OpenHuman's existing RPC layer. //! //! Everything the crate deliberately refused to own lives on this side of the @@ -29,10 +29,12 @@ //! is the safe direction: a missed retry costs a request, a wrong retry can //! cost a duplicate transaction. +use crate::openhuman::web3::wallet::primitives::rpc::{ + NetworkId, Transport, TransportError, TransportResult, +}; use async_trait::async_trait; use log::debug; use serde_json::Value; -use tinywallet::rpc::{NetworkId, Transport, TransportError, TransportResult}; use super::defaults::{rpc_url_for_chain, rpc_url_for_evm_network, EvmNetwork}; use super::ops::WalletChain; @@ -54,9 +56,10 @@ impl OpenHumanTransport { } /// Map a `tinywallet` network onto OpenHuman's chain enum plus a base URL. +#[allow(unreachable_patterns)] fn resolve(network: NetworkId) -> Result { match network.chain { - tinywallet::Chain::Evm => { + crate::openhuman::web3::wallet::primitives::Chain::Evm => { // An EVM request names its EIP-155 chain id; resolving it here is // what keeps `tinywallet` free of OpenHuman's network enum. let chain_id = network.evm_chain_id.ok_or_else(|| TransportError::Rpc { @@ -73,10 +76,16 @@ fn resolve(network: NetworkId) -> Result { })?; Ok(rpc_url_for_evm_network(evm)) } - tinywallet::Chain::Btc => Ok(rpc_url_for_chain(WalletChain::Btc)), - tinywallet::Chain::Solana => Ok(rpc_url_for_chain(WalletChain::Solana)), - tinywallet::Chain::Tron => Ok(rpc_url_for_chain(WalletChain::Tron)), - // `tinywallet::Chain` is `#[non_exhaustive]`, so a future variant must + crate::openhuman::web3::wallet::primitives::Chain::Btc => { + Ok(rpc_url_for_chain(WalletChain::Btc)) + } + crate::openhuman::web3::wallet::primitives::Chain::Solana => { + Ok(rpc_url_for_chain(WalletChain::Solana)) + } + crate::openhuman::web3::wallet::primitives::Chain::Tron => { + Ok(rpc_url_for_chain(WalletChain::Tron)) + } + // `crate::openhuman::web3::wallet::primitives::Chain` is `#[non_exhaustive]`, so a future variant must // be handled. Reporting it as authoritative is correct: no endpoint is // configured for it, and retrying elsewhere cannot change that. other => Err(TransportError::Rpc { @@ -237,7 +246,10 @@ mod tests { #[test] fn an_evm_request_without_a_chain_id_is_authoritative_not_retryable() { - let err = resolve(NetworkId::chain(tinywallet::Chain::Evm)).unwrap_err(); + let err = resolve(NetworkId::chain( + crate::openhuman::web3::wallet::primitives::Chain::Evm, + )) + .unwrap_err(); assert!(!err.is_retryable(), "{err}"); } @@ -252,9 +264,9 @@ mod tests { #[test] fn every_non_evm_chain_resolves() { for chain in [ - tinywallet::Chain::Btc, - tinywallet::Chain::Solana, - tinywallet::Chain::Tron, + crate::openhuman::web3::wallet::primitives::Chain::Btc, + crate::openhuman::web3::wallet::primitives::Chain::Solana, + crate::openhuman::web3::wallet::primitives::Chain::Tron, ] { assert!(resolve(NetworkId::chain(chain)).is_ok(), "{chain}"); } @@ -264,7 +276,7 @@ mod tests { fn transport_failures_are_retryable_and_everything_else_is_not() { // The conservative direction: only what this layer knows to be a // transport failure may drive a failover. - let network = NetworkId::chain(tinywallet::Chain::Btc); + let network = NetworkId::chain(crate::openhuman::web3::wallet::primitives::Chain::Btc); assert!( classify(network, "wallet RPC transport failed for x: refused".into()).is_retryable() ); diff --git a/src/openhuman/web3/x402/ops.rs b/src/openhuman/web3/x402/ops.rs index 16175e4b8c..91366f3ea1 100644 --- a/src/openhuman/web3/x402/ops.rs +++ b/src/openhuman/web3/x402/ops.rs @@ -595,7 +595,7 @@ pub(crate) fn build_evm_payment_with_signer( challenge: &PaymentRequired, req: &PaymentRequirements, ) -> Result { - use tinywallet::eip712; + use crate::openhuman::web3::wallet::primitives::eip712; let chain_id = req .evm_chain_id() @@ -695,7 +695,7 @@ pub(crate) fn build_evm_payment_with_signer( /// Derive the wallet's EVM signing key from the encrypted mnemonic. /// /// Returns the raw secret and the checksummed address it controls. Derivation -/// goes through `tinywallet::key` — the same BIP-32 walk the wallet domain uses, +/// goes through `crate::openhuman::web3::wallet::primitives::key` — the same BIP-32 walk the wallet domain uses, /// so an x402 payment is signed by exactly the account the wallet reports — and /// the key stays in this process. async fn derive_evm_signer() -> Result<(Vec, String), X402Error> { @@ -717,8 +717,8 @@ async fn derive_evm_signer() -> Result<(Vec, String), X402Error> { .map_err(|e| X402Error::Wallet(format!("decrypt mnemonic: {e}")))? .value; - let derived = tinywallet::key::derive( - tinywallet::Chain::Evm, + let derived = crate::openhuman::web3::wallet::primitives::key::derive( + crate::openhuman::web3::wallet::primitives::Chain::Evm, mnemonic.as_str(), &secret.derivation_path, ) @@ -732,7 +732,7 @@ async fn derive_evm_signer() -> Result<(Vec, String), X402Error> { /// The 20 raw bytes of an EVM address. fn evm_address_bytes(address: &str) -> Result<[u8; 20], X402Error> { - let validated = tinywallet::address::evm::validate(address) + let validated = crate::openhuman::web3::wallet::primitives::address::evm::validate(address) .map_err(|e| X402Error::Protocol(format!("invalid EVM address '{address}': {e}")))?; let body = validated.strip_prefix("0x").unwrap_or(&validated); let decoded = hex::decode(body) diff --git a/src/openhuman/web3/x402/x402_tests.rs b/src/openhuman/web3/x402/x402_tests.rs index 80d8d6efc3..d518653bf4 100644 --- a/src/openhuman/web3/x402/x402_tests.rs +++ b/src/openhuman/web3/x402/x402_tests.rs @@ -394,11 +394,11 @@ fn solana_payment_proof_serializes_correctly() { #[test] fn eip712_domain_separator_is_deterministic() { - // Now `tinywallet::eip712`, which also pins the hashes against the + // Now `crate::openhuman::web3::wallet::primitives::eip712`, which also pins the hashes against the // published EIP-712/EIP-3009 constants. What this still checks is the // property that matters at this layer: the separator binds the chain, so // an authorization cannot be replayed on another one. - use tinywallet::eip712::domain_separator; + use crate::openhuman::web3::wallet::primitives::eip712::domain_separator; let contract = base_usdc(); let sep1 = domain_separator(contract, 8453, "USD Coin", "2"); @@ -411,13 +411,17 @@ fn eip712_domain_separator_is_deterministic() { /// The BIP-39 vector mnemonic's EVM account: raw secret and its address. /// -/// Derived through `tinywallet::key`, which is what the production path uses, +/// Derived through `crate::openhuman::web3::wallet::primitives::key`, which is what the production path uses, /// so the test signs as exactly the account the wallet would. fn test_signer() -> (Vec, String) { let test_mnemonic = "abandon abandon abandon abandon abandon abandon \ abandon abandon abandon abandon abandon about"; - let derived = - tinywallet::key::derive(tinywallet::Chain::Evm, test_mnemonic, "m/44'/60'/0'/0/0").unwrap(); + let derived = crate::openhuman::web3::wallet::primitives::key::derive( + crate::openhuman::web3::wallet::primitives::Chain::Evm, + test_mnemonic, + "m/44'/60'/0'/0/0", + ) + .unwrap(); ( derived.secret_bytes().to_vec(), derived.address().to_string(), @@ -436,7 +440,9 @@ fn address_bytes(hex: &str) -> [u8; 20] { #[test] fn eip3009_struct_hash_is_deterministic() { - use tinywallet::eip712::{transfer_with_authorization_hash, u256_from_u64}; + use crate::openhuman::web3::wallet::primitives::eip712::{ + transfer_with_authorization_hash, u256_from_u64, + }; let from = address_bytes(&"aa".repeat(20)); let to = address_bytes(&"bb".repeat(20)); @@ -532,7 +538,7 @@ fn build_evm_payment_with_test_key_produces_valid_payload() { assert_eq!(evm.authorization.value, "2500"); assert_eq!(evm.authorization.valid_after, "0"); assert!(evm.authorization.nonce.starts_with("0x")); - // Checksummed, as `tinywallet::address::evm` renders it, and as + // Checksummed, as `crate::openhuman::web3::wallet::primitives::address::evm` renders it, and as // the requirement itself carried it. assert_eq!( evm.authorization.to, @@ -540,8 +546,8 @@ fn build_evm_payment_with_test_key_produces_valid_payload() { ); assert_eq!(evm.authorization.from, from_address); + use crate::openhuman::web3::wallet::primitives::eip712; use k256::ecdsa::{RecoveryId, Signature, VerifyingKey}; - use tinywallet::eip712; let raw = hex::decode(evm.signature.trim_start_matches("0x")).unwrap(); assert!(matches!(raw[64], 27 | 28), "invalid recovery byte"); diff --git a/tests/json_rpc_e2e.rs b/tests/json_rpc_e2e.rs index 99d08bf6dd..ef104e2d23 100644 --- a/tests/json_rpc_e2e.rs +++ b/tests/json_rpc_e2e.rs @@ -92,9 +92,12 @@ fn ensure_json_rpc_e2e_memory_seams() { .name("json-rpc-e2e-memory-seams".to_string()) .stack_size(8 * 1024 * 1024) .spawn(|| { - openhuman_core::openhuman::memory::host_impls::install_memory_host_seams(Arc::new( - openhuman_core::openhuman::config::Config::default(), - )); + let config = Arc::new(openhuman_core::openhuman::config::Config::default()); + openhuman_core::openhuman::memory::host_impls::install_memory_host_seams( + config.clone(), + ); + #[cfg(feature = "modules")] + openhuman_core::openhuman::modules::memory::set_modules_policy(config); }) .expect("spawn json_rpc e2e memory seam installer") .join() diff --git a/tests/memory_golden_fixture_e2e.rs b/tests/memory_golden_fixture_e2e.rs index c6f5f68820..fe6114daba 100644 --- a/tests/memory_golden_fixture_e2e.rs +++ b/tests/memory_golden_fixture_e2e.rs @@ -133,15 +133,24 @@ fn env_lock() -> std::sync::MutexGuard<'static, ()> { /// This integration target binds the transport-independent global memory /// client directly, so it must provide the same host seams that normal core /// startup installs before opening memory stores. -fn ensure_memory_seams() { +fn ensure_memory_seams(workspace: &Path) { MEMORY_SEAMS_INIT.get_or_init(|| { + let workspace = workspace.to_path_buf(); std::thread::Builder::new() .name("memory-golden-fixture-seams".to_string()) .stack_size(8 * 1024 * 1024) - .spawn(|| { - openhuman_core::openhuman::memory::host_impls::install_memory_host_seams(Arc::new( - openhuman_core::openhuman::config::Config::default(), - )); + .spawn(move || { + let config = Arc::new(openhuman_core::openhuman::config::Config { + workspace_dir: workspace.clone(), + action_dir: workspace.clone(), + config_path: workspace.join("config.toml"), + ..openhuman_core::openhuman::config::Config::default() + }); + openhuman_core::openhuman::memory::host_impls::install_memory_host_seams( + config.clone(), + ); + #[cfg(feature = "modules")] + openhuman_core::openhuman::modules::memory::set_modules_policy(config); }) .expect("spawn golden fixture memory seam installer") .join() @@ -263,12 +272,12 @@ async fn fresh_workspace_schema_matches_the_committed_manifest() { #[tokio::test] async fn golden_fixture_rows_read_back_and_schema_is_stable_after_reopen() { let _lock = env_lock(); - ensure_memory_seams(); let tmp = tempdir().expect("tempdir"); let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); let workspace = tmp.path().join("workspace"); copy_fixture_to(&workspace); let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + ensure_memory_seams(&workspace); let before = golden::schema_manifest(&workspace).expect("dump schema before open"); @@ -402,11 +411,11 @@ fn run_second_process_readback(workspace: &Path) { #[tokio::test] #[ignore = "spawned as a child process by golden_fixture_rows_read_back_and_schema_is_stable_after_reopen"] async fn second_process_readback() { - ensure_memory_seams(); let Ok(workspace) = std::env::var(SECOND_PROCESS_WS_ENV) else { panic!("{SECOND_PROCESS_WS_ENV} not set — this test is spawned, not run directly"); }; let workspace = PathBuf::from(workspace); + ensure_memory_seams(&workspace); eprintln!("[golden-fixture][child] reopening {}", workspace.display()); openhuman_core::openhuman::memory::global::init(workspace.clone()) @@ -459,12 +468,12 @@ fn prune_non_db_files(dir: &Path) { #[ignore = "regenerates the committed golden fixture; run via scripts/regen-memory-golden-fixture.sh"] async fn regenerate_golden_fixture() { let _lock = env_lock(); - ensure_memory_seams(); let tmp = tempdir().expect("tempdir"); let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); let staging = tmp.path().join("workspace"); std::fs::create_dir_all(&staging).expect("create staging workspace"); let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &staging); + ensure_memory_seams(&staging); openhuman_core::openhuman::memory::global::init(staging.clone()) .expect("bind global memory client to the staging workspace"); diff --git a/tests/memory_golden_parity_e2e.rs b/tests/memory_golden_parity_e2e.rs index 4f410a1460..def03839b1 100644 --- a/tests/memory_golden_parity_e2e.rs +++ b/tests/memory_golden_parity_e2e.rs @@ -112,15 +112,24 @@ fn env_lock() -> std::sync::MutexGuard<'static, ()> { /// This target calls the memory operations directly rather than through a core /// runtime, so install the host seams that normal startup wires first. -fn ensure_memory_seams() { +fn ensure_memory_seams(workspace: &Path) { MEMORY_SEAMS_INIT.get_or_init(|| { + let workspace = workspace.to_path_buf(); std::thread::Builder::new() .name("memory-golden-parity-seams".to_string()) .stack_size(8 * 1024 * 1024) - .spawn(|| { - openhuman_core::openhuman::memory::host_impls::install_memory_host_seams(Arc::new( - Config::default(), - )); + .spawn(move || { + let config = Arc::new(Config { + workspace_dir: workspace.clone(), + action_dir: workspace.clone(), + config_path: workspace.join("config.toml"), + ..Config::default() + }); + openhuman_core::openhuman::memory::host_impls::install_memory_host_seams( + config.clone(), + ); + #[cfg(feature = "modules")] + openhuman_core::openhuman::modules::memory::set_modules_policy(config); }) .expect("spawn golden parity memory seam installer") .join() @@ -389,12 +398,12 @@ async fn init_and_scan(ns: &str, workspace: &Path) -> BTreeSet { #[tokio::test] async fn golden_workspace_composes_substrate_and_unified_tiers() { let _lock = env_lock(); - ensure_memory_seams(); let tmp = tempdir().expect("tempdir"); let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); let workspace = tmp.path().join("workspace"); std::fs::create_dir_all(&workspace).expect("mkdir workspace"); let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace); + ensure_memory_seams(&workspace); let tables = init_and_scan("golden-parity-e2e", &workspace).await; diff --git a/tests/memory_roundtrip_e2e.rs b/tests/memory_roundtrip_e2e.rs index 857c8b3b4f..6c7499fce7 100644 --- a/tests/memory_roundtrip_e2e.rs +++ b/tests/memory_roundtrip_e2e.rs @@ -53,6 +53,7 @@ impl Drop for EnvVarGuard { /// Serialises tests: `HOME` + `OPENHUMAN_WORKSPACE` are process-global. static ENV_LOCK: OnceLock> = OnceLock::new(); static MEMORY_SEAMS_INIT: OnceLock<()> = OnceLock::new(); +static TEST_ROOT: OnceLock = OnceLock::new(); fn env_lock() -> std::sync::MutexGuard<'static, ()> { match ENV_LOCK.get_or_init(|| Mutex::new(())).lock() { @@ -63,15 +64,28 @@ fn env_lock() -> std::sync::MutexGuard<'static, ()> { /// This integration target calls memory operations without constructing a core /// runtime, so it supplies the seams that normal startup installs first. -fn ensure_memory_seams() { +fn test_root() -> &'static tempfile::TempDir { + TEST_ROOT.get_or_init(|| tempdir().expect("memory roundtrip tempdir")) +} + +fn ensure_memory_seams(workspace: &Path) { MEMORY_SEAMS_INIT.get_or_init(|| { + let workspace = workspace.to_path_buf(); std::thread::Builder::new() .name("memory-roundtrip-seams".to_string()) .stack_size(8 * 1024 * 1024) - .spawn(|| { - openhuman_core::openhuman::memory::host_impls::install_memory_host_seams(Arc::new( - openhuman_core::openhuman::config::Config::default(), - )); + .spawn(move || { + let config = Arc::new(openhuman_core::openhuman::config::Config { + workspace_dir: workspace.clone(), + action_dir: workspace.clone(), + config_path: workspace.join("config.toml"), + ..openhuman_core::openhuman::config::Config::default() + }); + openhuman_core::openhuman::memory::host_impls::install_memory_host_seams( + config.clone(), + ); + #[cfg(feature = "modules")] + openhuman_core::openhuman::modules::memory::set_modules_policy(config); }) .expect("spawn memory roundtrip seam installer") .join() @@ -126,12 +140,12 @@ fn recall_context_request() -> RecallContextRequest { #[tokio::test] async fn doc_put_then_recall_memories_returns_canary() { let _lock = env_lock(); - ensure_memory_seams(); - let tmp = tempdir().expect("tempdir"); + let tmp = test_root(); let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); let workspace_path = tmp.path().join("workspace"); std::fs::create_dir_all(&workspace_path).expect("create workspace dir"); let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace_path); + ensure_memory_seams(&workspace_path); // Store the canary document. let put_outcome = doc_put(put_params()).await.expect("doc_put rpc"); @@ -157,12 +171,12 @@ async fn doc_put_then_recall_memories_returns_canary() { #[tokio::test] async fn doc_put_then_recall_context_renders_llm_context_message() { let _lock = env_lock(); - ensure_memory_seams(); - let tmp = tempdir().expect("tempdir"); + let tmp = test_root(); let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); let workspace_path = tmp.path().join("workspace"); std::fs::create_dir_all(&workspace_path).expect("create workspace dir"); let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace_path); + ensure_memory_seams(&workspace_path); doc_put(put_params()).await.expect("doc_put rpc"); @@ -193,12 +207,12 @@ async fn doc_put_then_recall_context_renders_llm_context_message() { #[tokio::test] async fn doc_put_with_multibyte_at_body_preview_boundary_does_not_panic() { let _lock = env_lock(); - ensure_memory_seams(); - let tmp = tempdir().expect("tempdir"); + let tmp = test_root(); let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); let workspace_path = tmp.path().join("workspace"); std::fs::create_dir_all(&workspace_path).expect("create workspace dir"); let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace_path); + ensure_memory_seams(&workspace_path); const BODY_PREVIEW_MAX_BYTES: usize = 2048; let zwnj = '\u{200c}'; // 3-byte codepoint @@ -252,12 +266,12 @@ async fn doc_put_with_multibyte_at_body_preview_boundary_does_not_panic() { #[tokio::test] async fn clear_namespace_removes_canary_from_recall() { let _lock = env_lock(); - ensure_memory_seams(); - let tmp = tempdir().expect("tempdir"); + let tmp = test_root(); let _home = EnvVarGuard::set_to_path("HOME", tmp.path()); let workspace_path = tmp.path().join("workspace"); std::fs::create_dir_all(&workspace_path).expect("create workspace dir"); let _ws = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", &workspace_path); + ensure_memory_seams(&workspace_path); // Seed the namespace. doc_put(put_params()).await.expect("seed doc_put"); diff --git a/tests/memory_sources_e2e.rs b/tests/memory_sources_e2e.rs index 2ae5667781..a147980883 100644 --- a/tests/memory_sources_e2e.rs +++ b/tests/memory_sources_e2e.rs @@ -21,6 +21,7 @@ const TEST_RPC_TOKEN: &str = "memory-sources-e2e-token"; static AUTH_INIT: OnceLock<()> = OnceLock::new(); static ENV_LOCK: OnceLock> = OnceLock::new(); static MEMORY_SEAMS_INIT: OnceLock<()> = OnceLock::new(); +static TEST_HOME: OnceLock = OnceLock::new(); fn env_lock() -> std::sync::MutexGuard<'static, ()> { let mutex = ENV_LOCK.get_or_init(|| Mutex::new(())); @@ -46,9 +47,12 @@ fn ensure_memory_seams() { .name("memory-sources-e2e-seams".to_string()) .stack_size(8 * 1024 * 1024) .spawn(|| { - openhuman_core::openhuman::memory::host_impls::install_memory_host_seams(Arc::new( - openhuman_core::openhuman::config::Config::default(), - )); + let config = Arc::new(openhuman_core::openhuman::config::Config::default()); + openhuman_core::openhuman::memory::host_impls::install_memory_host_seams( + config.clone(), + ); + #[cfg(feature = "modules")] + openhuman_core::openhuman::modules::memory::set_modules_policy(config); }) .expect("spawn memory sources seam installer") .join() @@ -56,6 +60,12 @@ fn ensure_memory_seams() { }); } +fn test_home() -> &'static Path { + TEST_HOME + .get_or_init(|| tempdir().expect("memory sources tempdir")) + .path() +} + struct EnvVarGuard { key: &'static str, old: Option, @@ -164,8 +174,7 @@ fn ok(v: &Value, ctx: &str) -> Value { #[tokio::test] async fn memory_sources_crud_and_folder_read_flow() { let _guard = env_lock(); - let tmp = tempdir().expect("tempdir"); - let home = tmp.path(); + let home = test_home(); let openhuman_home = home.join(".openhuman"); let _home = EnvVarGuard::set_to_path("HOME", home); @@ -437,8 +446,7 @@ async fn memory_sources_crud_and_folder_read_flow() { #[tokio::test] async fn memory_sources_validation_rejects_bad_input() { let _guard = env_lock(); - let tmp = tempdir().expect("tempdir"); - let home = tmp.path(); + let home = test_home(); let openhuman_home = home.join(".openhuman"); let _home = EnvVarGuard::set_to_path("HOME", home); @@ -518,8 +526,7 @@ async fn memory_sources_github_repo_activity_flow() { return; } let _guard = env_lock(); - let tmp = tempdir().expect("tempdir"); - let home = tmp.path(); + let home = test_home(); let openhuman_home = home.join(".openhuman"); let _home = EnvVarGuard::set_to_path("HOME", home); @@ -699,8 +706,7 @@ async fn memory_sources_github_repo_activity_flow() { #[tokio::test] async fn memory_sources_composio_registry_flow() { let _guard = env_lock(); - let tmp = tempdir().expect("tempdir"); - let home = tmp.path(); + let home = test_home(); let openhuman_home = home.join(".openhuman"); let _home = EnvVarGuard::set_to_path("HOME", home); diff --git a/tests/raw_coverage/memory_sync_slack_bus_raw_coverage_e2e.rs b/tests/raw_coverage/memory_sync_slack_bus_raw_coverage_e2e.rs index a4edf00905..de5687fcbf 100644 --- a/tests/raw_coverage/memory_sync_slack_bus_raw_coverage_e2e.rs +++ b/tests/raw_coverage/memory_sync_slack_bus_raw_coverage_e2e.rs @@ -15,11 +15,7 @@ use serde_json::{json, Value}; use tempfile::TempDir; use openhuman_core::core::events::DomainEvent; -use tinybus::EventHandler; use openhuman_core::openhuman::config::Config; -use openhuman_core::openhuman::security::credentials::{ - AuthService, APP_SESSION_PROVIDER, DEFAULT_AUTH_PROFILE_NAME, -}; use openhuman_core::openhuman::memory::global as memory_global; use openhuman_core::openhuman::memory::sync::composio::bus::{ ComposioConfigChangedSubscriber, ComposioConnectionCreatedSubscriber, ComposioTriggerSubscriber, @@ -31,19 +27,25 @@ use openhuman_core::openhuman::memory::sync::composio::providers::slack::{ use openhuman_core::openhuman::memory::sync::composio::providers::{ ComposioProvider, ProviderContext, SyncReason, }; +use openhuman_core::openhuman::security::credentials::{ + AuthService, APP_SESSION_PROVIDER, DEFAULT_AUTH_PROFILE_NAME, +}; +use tinybus::EventHandler; static ENV_LOCK: &OnceLock> = &crate::SHARED_ENV_LOCK; static MEMORY_SEAMS_INIT: OnceLock<()> = OnceLock::new(); -fn ensure_memory_seams() { +fn ensure_memory_seams(config: Arc) { MEMORY_SEAMS_INIT.get_or_init(|| { std::thread::Builder::new() .name("memory-sync-slack-bus-raw-coverage-seams".to_string()) .stack_size(8 * 1024 * 1024) - .spawn(|| { + .spawn(move || { openhuman_core::openhuman::memory::host_impls::install_memory_host_seams( - Arc::new(Config::default()), + Arc::clone(&config), ); + #[cfg(feature = "modules")] + openhuman_core::openhuman::modules::memory::set_modules_policy(config); }) .expect("spawn slack bus memory seam installer") .join() @@ -91,7 +93,6 @@ impl Drop for EnvGuard { } fn config_in(tmp: &TempDir) -> Config { - ensure_memory_seams(); let mut config = Config { config_path: tmp.path().join("config.toml"), workspace_dir: tmp.path().join("workspace"), @@ -99,6 +100,7 @@ fn config_in(tmp: &TempDir) -> Config { ..Config::default() }; config.secrets.encrypt = false; + ensure_memory_seams(Arc::new(config.clone())); config } diff --git a/tests/raw_coverage/memory_threads_raw_coverage_e2e.rs b/tests/raw_coverage/memory_threads_raw_coverage_e2e.rs index 9255d5306b..2aaae2553e 100644 --- a/tests/raw_coverage/memory_threads_raw_coverage_e2e.rs +++ b/tests/raw_coverage/memory_threads_raw_coverage_e2e.rs @@ -21,41 +21,18 @@ use openhuman_core::openhuman::agent::progress::AgentProgress; use openhuman_core::openhuman::agent::task_board::{TaskBoard, TaskBoardCard, TaskCardStatus}; use openhuman_core::openhuman::config::Config; use openhuman_core::openhuman::inference::embeddings::NoopEmbedding; +use openhuman_core::openhuman::memory::api::tool_memory::{ + ToolMemoryPriority as ApiToolMemoryPriority, ToolMemoryRule as ApiToolMemoryRule, + ToolMemorySource as ApiToolMemorySource, +}; use openhuman_core::openhuman::memory::query::{ MemoryQueryTool, MemoryTreeDrillDownTool, MemoryTreeFetchLeavesTool, MemoryTreeIngestDocumentTool, MemoryTreeQuerySourceTool, MemoryTreeSearchEntitiesTool, }; -use openhuman_core::openhuman::memory::tools::{ - MemoryForgetTool, MemoryRecallTool, MemoryStoreTool, -}; -use openhuman_core::openhuman::memory::tree_policy::TreePolicy; -use openhuman_core::openhuman::memory::tree_source; -use openhuman_core::openhuman::memory::{ - all_memory_controller_schemas, all_memory_registered_controllers, - preferences::{ - load_general_preferences, recall_related_preferences, recall_situational_preferences, - USER_PREF_GENERAL_NAMESPACE, USER_PREF_SITUATIONAL_NAMESPACE, - }, - read_rpc as memory_read_rpc, - remember::RememberSourceKind, - rpc_models::{ - ApiEnvelope, ApiError, ApiMeta, AppendConversationMessageRequest, - ConversationMessageRecord, ConversationMessagesRequest, CreateConversationThreadRequest, - DeleteConversationThreadRequest, DeleteDocumentRequest, EmptyRequest, - GenerateConversationThreadTitleRequest, ListDocumentsRequest, ListMemoryFilesRequest, - MemoryInitRequest, PaginationMeta, QueryNamespaceRequest, ReadMemoryFileRequest, - RecallContextRequest, RecallMemoriesRequest, UpdateConversationMessageRequest, - UpdateConversationThreadLabelsRequest, UpdateConversationThreadTitleRequest, - UpsertConversationThreadRequest, WriteMemoryFileRequest, - }, - traits::{Memory, MemoryCategory, MemoryEntry, NamespaceSummary, RecallOpts}, - util::redact::{redact, redact_endpoint}, - MemoryIngestionConfig, MemoryIngestionRequest, -}; use openhuman_core::openhuman::memory::queue::types::ReembedBackfillPayload; use openhuman_core::openhuman::memory::queue::{ - self as memory_queue, AppendBufferPayload, AppendTarget, ExtractChunkPayload, FlushStalePayload, JobKind, - JobStatus, NewJob, NodeRef, SealPayload, DEFAULT_LOCK_DURATION_MS, + self as memory_queue, AppendBufferPayload, AppendTarget, ExtractChunkPayload, + FlushStalePayload, JobKind, JobStatus, NewJob, NodeRef, SealPayload, DEFAULT_LOCK_DURATION_MS, }; use openhuman_core::openhuman::memory::sources::readers::reader_for; use openhuman_core::openhuman::memory::sources::registry; @@ -79,16 +56,6 @@ use openhuman_core::openhuman::memory::store::trees::types::{ use openhuman_core::openhuman::memory::store::{ MemoryClient, NamespaceDocumentInput, UnifiedMemory, }; -use tinycortex::memory::ingest::canonicalize::chat::{ - canonicalise as canonicalise_chat, ChatBatch, ChatMessage, -}; -use tinycortex::memory::ingest::canonicalize::document::{ - canonicalise as canonicalise_document, DocumentInput, -}; -use tinycortex::memory::ingest::canonicalize::email::{ - canonicalise as canonicalise_email, EmailMessage, EmailThread, -}; -use tinycortex::memory::ingest::canonicalize::email_clean; use openhuman_core::openhuman::memory::sync::composio; use openhuman_core::openhuman::memory::sync::composio::providers::profile::{ canonicalize, delete_connected_identity_facets, is_self_identity, is_self_identity_any_toolkit, @@ -117,16 +84,18 @@ use openhuman_core::openhuman::memory::sync::composio::providers::{ use openhuman_core::openhuman::memory::sync::sync_status::{ rpc as memory_sync_status_rpc, schemas as memory_sync_status_schemas, }; -use tinycortex::memory::sync::{SyncOutcome as PipelineSyncOutcome, SyncPipelineKind}; -use openhuman_core::openhuman::memory::tools::tool_memory::{ - MemoryToolsListTool, MemoryToolsPutTool, +use openhuman_core::openhuman::memory::tool_memory::prompt::{ + render_tool_memory_rules, ToolMemoryRulesSection, TOOL_MEMORY_HEADING, }; use openhuman_core::openhuman::memory::tool_memory::{ - tool_memory_namespace, tool_memory_store, ToolMemoryPriority, ToolMemoryRule, - ToolMemorySource, TOOL_MEMORY_PROMPT_CAP, + tool_memory_namespace, tool_memory_store, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource, + TOOL_MEMORY_PROMPT_CAP, }; -use openhuman_core::openhuman::memory::tool_memory::prompt::{ - render_tool_memory_rules, ToolMemoryRulesSection, TOOL_MEMORY_HEADING, +use openhuman_core::openhuman::memory::tools::tool_memory::{ + MemoryToolsListTool, MemoryToolsPutTool, +}; +use openhuman_core::openhuman::memory::tools::{ + MemoryForgetTool, MemoryRecallTool, MemoryStoreTool, }; use openhuman_core::openhuman::memory::tree::score::embed::Embedder; use openhuman_core::openhuman::memory::tree::score::extract::{ @@ -152,6 +121,30 @@ use openhuman_core::openhuman::memory::tree::tree_runtime::{ NodeLevel, TreeNode, }; use openhuman_core::openhuman::memory::tree::{retrieval, score::embed}; +use openhuman_core::openhuman::memory::tree_policy::TreePolicy; +use openhuman_core::openhuman::memory::tree_source; +use openhuman_core::openhuman::memory::{ + all_memory_controller_schemas, all_memory_registered_controllers, + preferences::{ + load_general_preferences, recall_related_preferences, recall_situational_preferences, + USER_PREF_GENERAL_NAMESPACE, USER_PREF_SITUATIONAL_NAMESPACE, + }, + read_rpc as memory_read_rpc, + remember::RememberSourceKind, + rpc_models::{ + ApiEnvelope, ApiError, ApiMeta, AppendConversationMessageRequest, + ConversationMessageRecord, ConversationMessagesRequest, CreateConversationThreadRequest, + DeleteConversationThreadRequest, DeleteDocumentRequest, EmptyRequest, + GenerateConversationThreadTitleRequest, ListDocumentsRequest, ListMemoryFilesRequest, + MemoryInitRequest, PaginationMeta, QueryNamespaceRequest, ReadMemoryFileRequest, + RecallContextRequest, RecallMemoriesRequest, UpdateConversationMessageRequest, + UpdateConversationThreadLabelsRequest, UpdateConversationThreadTitleRequest, + UpsertConversationThreadRequest, WriteMemoryFileRequest, + }, + traits::{Memory, MemoryCategory, MemoryEntry, NamespaceSummary, RecallOpts}, + util::redact::{redact, redact_endpoint}, + MemoryIngestionConfig, MemoryIngestionRequest, +}; use openhuman_core::openhuman::security::{AutonomyLevel, SecurityPolicy}; use openhuman_core::openhuman::threads::ops as thread_ops; use openhuman_core::openhuman::threads::title::{ @@ -168,6 +161,17 @@ use openhuman_core::openhuman::threads::{ all_threads_controller_schemas, all_threads_registered_controllers, }; use openhuman_core::openhuman::tools::traits::{PermissionLevel, Tool, ToolCategory}; +use tinycortex::memory::ingest::canonicalize::chat::{ + canonicalise as canonicalise_chat, ChatBatch, ChatMessage, +}; +use tinycortex::memory::ingest::canonicalize::document::{ + canonicalise as canonicalise_document, DocumentInput, +}; +use tinycortex::memory::ingest::canonicalize::email::{ + canonicalise as canonicalise_email, EmailMessage, EmailThread, +}; +use tinycortex::memory::ingest::canonicalize::email_clean; +use tinycortex::memory::sync::{SyncOutcome as PipelineSyncOutcome, SyncPipelineKind}; struct EnvVarGuard { key: &'static str, @@ -1678,8 +1682,9 @@ fn memory_tree_runtime_store_buffers_and_retrieval_wire_helpers() { openhuman_core::openhuman::memory::tree::tree::TreeFactory::from_tree(&source_tree).kind(), TreeKind::Source ); - let topic_factory = - openhuman_core::openhuman::memory::tree::tree::TreeFactory::topic("email:alice@example.com"); + let topic_factory = openhuman_core::openhuman::memory::tree::tree::TreeFactory::topic( + "email:alice@example.com", + ); assert!(matches!( topic_factory.summary_tree_kind(), openhuman_core::openhuman::memory::store::content::SummaryTreeKind::Topic @@ -2533,7 +2538,14 @@ async fn memory_queue_and_tool_memory_public_stores_cover_persistence_edges() { .iter() .all(|rule| rule.priority.is_eager())); assert_eq!(TOOL_MEMORY_PROMPT_CAP, 30); - let rendered = render_tool_memory_rules(&[normal.clone(), updated.clone(), high.clone()]); + let render_rules: Vec = [normal.clone(), updated.clone(), high.clone()] + .into_iter() + .map(|rule| { + serde_json::from_value(serde_json::to_value(rule).expect("serialize tool rule")) + .expect("convert tool rule to host API") + }) + .collect(); + let rendered = render_tool_memory_rules(&render_rules); assert!(rendered.starts_with(TOOL_MEMORY_HEADING)); assert!(rendered.find("**[critical]**") < rendered.find("**[high]**")); assert!(rendered.contains("### `shell`")); @@ -2657,16 +2669,15 @@ fn memory_tree_io_contract_types_round_trip_leaf_read_and_write_shapes() { assert_eq!(round_trip.content, payload.content); assert_eq!(round_trip.score, payload.score); - let write_default_json = serde_json::to_value( - openhuman_core::openhuman::memory::tree::TreeWriteRequest { + let write_default_json = + serde_json::to_value(openhuman_core::openhuman::memory::tree::TreeWriteRequest { tree_id: "tree-contract".into(), tree_kind: TreeKind::Source, leaf: round_trip.clone(), label_strategy: Default::default(), deferred: false, - }, - ) - .expect("write request json"); + }) + .expect("write request json"); assert_eq!(write_default_json["label_strategy"], "inherit"); assert_eq!(write_default_json["deferred"], false); @@ -2954,7 +2965,6 @@ impl ComposioProvider for RawCoverageProvider { }) } } - } struct EmptySlugProvider; @@ -2971,7 +2981,6 @@ impl ComposioProvider for EmptySlugProvider { ) -> Result { Ok(ProviderUserProfile::default()) } - } #[tokio::test] @@ -3980,6 +3989,12 @@ async fn memory_ops_public_handlers_cover_document_file_kv_graph_and_envelopes_b ensure_memory_seams(); let tmp = TempDir::new().expect("tempdir"); let _workspace = EnvVarGuard::set_to_path("OPENHUMAN_WORKSPACE", tmp.path()); + #[cfg(feature = "modules")] + { + let mut config = Config::default(); + config.workspace_dir = tmp.path().to_path_buf(); + openhuman_core::openhuman::modules::memory::set_modules_policy(Arc::new(config)); + } let init = openhuman_core::openhuman::memory::ops::memory_init(MemoryInitRequest { jwt_token: Some("ignored-token".into()), @@ -4237,8 +4252,8 @@ async fn memory_ops_public_handlers_cover_document_file_kv_graph_and_envelopes_b openhuman_core::openhuman::memory::ops::ToolRulePutParams { tool_name: "shell".into(), rule: "Use dry-run flags before changing files.".into(), - priority: Some(ToolMemoryPriority::High), - source: Some(ToolMemorySource::UserExplicit), + priority: Some(ApiToolMemoryPriority::High), + source: Some(ApiToolMemorySource::UserExplicit), tags: vec!["safety".into()], id: Some("ops-rule-1".into()), }, @@ -4247,7 +4262,7 @@ async fn memory_ops_public_handlers_cover_document_file_kv_graph_and_envelopes_b .expect("tool rule put") .value; assert_eq!(tool_rule.id, "ops-rule-1"); - assert_eq!(tool_rule.priority, ToolMemoryPriority::High); + assert_eq!(tool_rule.priority, ApiToolMemoryPriority::High); let fetched_rule = openhuman_core::openhuman::memory::ops::tool_rule_get( openhuman_core::openhuman::memory::ops::ToolRuleRefParams { tool_name: "shell".into(), @@ -4551,24 +4566,26 @@ async fn tree_summarizer_ops_cover_validation_query_and_local_provider_guards() assert!(empty_content.contains("content must not be empty")); let ts = Utc.with_ymd_and_hms(2026, 5, 29, 17, 0, 0).unwrap(); - let ingest = openhuman_core::openhuman::memory::tree::tree_runtime::ops::tree_summarizer_ingest( - &config, - " ops_ns ", - "buffered raw content for summarizer ops", - Some(ts), - Some(&json!({ "source": "coverage" })), - ) - .await - .expect("ingest buffer"); + let ingest = + openhuman_core::openhuman::memory::tree::tree_runtime::ops::tree_summarizer_ingest( + &config, + " ops_ns ", + "buffered raw content for summarizer ops", + Some(ts), + Some(&json!({ "source": "coverage" })), + ) + .await + .expect("ingest buffer"); assert_eq!(ingest.value["buffered"], true); assert_eq!(ingest.value["namespace"], "ops_ns"); assert_eq!(ingest.value["has_metadata"], true); - let status = openhuman_core::openhuman::memory::tree::tree_runtime::ops::tree_summarizer_status( - &config, "ops_ns", - ) - .await - .expect("status"); + let status = + openhuman_core::openhuman::memory::tree::tree_runtime::ops::tree_summarizer_status( + &config, "ops_ns", + ) + .await + .expect("status"); assert_eq!(status.value["namespace"], "ops_ns"); assert_eq!(status.value["total_nodes"], 0); @@ -4582,13 +4599,14 @@ async fn tree_summarizer_ops_cover_validation_query_and_local_provider_guards() assert_eq!(query.value["node"]["node_id"], "root"); assert!(query.logs[0].contains("queried node 'root'")); - let missing = openhuman_core::openhuman::memory::tree::tree_runtime::ops::tree_summarizer_query( - &config, - "ops_ns", - Some("2026/05/29/17"), - ) - .await - .unwrap_err(); + let missing = + openhuman_core::openhuman::memory::tree::tree_runtime::ops::tree_summarizer_query( + &config, + "ops_ns", + Some("2026/05/29/17"), + ) + .await + .unwrap_err(); assert!(missing.contains("node '2026/05/29/17' not found")); let provider_guard = @@ -4689,7 +4707,8 @@ async fn memory_sources_types_registry_and_sync_state_cover_public_persistence_e MemoryClient::from_workspace_dir(tmp.path().join("memory-sync-state")) .expect("memory client"), ); - let adapter = openhuman_core::openhuman::memory::tinycortex::HostSyncAdapter::new(memory.clone()); + let adapter = + openhuman_core::openhuman::memory::tinycortex::HostSyncAdapter::new(memory.clone()); let fresh = SyncState::load(&adapter, "gmail", "conn-raw") .await .expect("fresh state"); diff --git a/tests/raw_coverage/near90_closure_raw_coverage_e2e.rs b/tests/raw_coverage/near90_closure_raw_coverage_e2e.rs index c198885bc7..ff7a6c0d22 100644 --- a/tests/raw_coverage/near90_closure_raw_coverage_e2e.rs +++ b/tests/raw_coverage/near90_closure_raw_coverage_e2e.rs @@ -105,14 +105,16 @@ fn env_lock() -> std::sync::MutexGuard<'static, ()> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } -fn ensure_memory_seams() { +fn ensure_memory_seams(config: Arc) { std::thread::Builder::new() .name("round20-memory-seams".to_string()) .stack_size(8 * 1024 * 1024) - .spawn(|| { - openhuman_core::openhuman::memory::host_impls::install_memory_host_seams(Arc::new( - openhuman_core::openhuman::config::Config::default(), + .spawn(move || { + openhuman_core::openhuman::memory::host_impls::install_memory_host_seams(Arc::clone( + &config, )); + #[cfg(feature = "modules")] + openhuman_core::openhuman::modules::memory::set_modules_policy(config); }) .expect("spawn round20 memory seam installer") .join() @@ -508,8 +510,8 @@ async fn round20_memory_sources_readers_and_sync_cover_error_edges_without_netwo #[tokio::test] async fn round20_memory_documents_files_and_envelopes_cover_success_and_failure_paths() { let _lock = env_lock(); - ensure_memory_seams(); let harness = setup("http://127.0.0.1:9"); + ensure_memory_seams(Arc::new(harness.config().await)); let init = memory_init(MemoryInitRequest { jwt_token: Some("ignored-round20".to_string()), diff --git a/vendor/tinydocs b/vendor/tinydocs deleted file mode 160000 index 7c907265fb..0000000000 --- a/vendor/tinydocs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7c907265fbc99c45397676202047ab2ac84e8643 diff --git a/vendor/tinymemory b/vendor/tinymemory index 82c2a210e8..1e2338aa71 160000 --- a/vendor/tinymemory +++ b/vendor/tinymemory @@ -1 +1 @@ -Subproject commit 82c2a210e8f537f8328a4d3022a87baf6aa20f4e +Subproject commit 1e2338aa71cf979915c749297b0fb77706e80ac8 diff --git a/vendor/tinywallet b/vendor/tinywallet deleted file mode 160000 index f82edabb47..0000000000 --- a/vendor/tinywallet +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f82edabb47ff3ea08a155db2c7c66d7cce7f8594