diff --git a/.github/workflows/ci-lite.yml b/.github/workflows/ci-lite.yml index 0e3b89a1a1..32c7bc0cb1 100644 --- a/.github/workflows/ci-lite.yml +++ b/.github/workflows/ci-lite.yml @@ -515,7 +515,7 @@ jobs: # # This asserts the calibration still holds. If it fails, every # projection built on the simulator is suspect until it is fixed. - run: python3 scripts/dep-sim.py --cut-nothing --expect-names 284 + run: python3 scripts/dep-sim.py --cut-nothing --expect-names 281 - name: Guard — new feature-gated test modules must be acknowledged # Self-maintaining coverage: the set of source files that #[cfg]-gate a test on @@ -681,6 +681,45 @@ jobs: - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov + - name: Install checksum-pinned native test modules + run: | + # tinybus validates every directory ancestor. GitHub mounts the + # checkout as the host runner uid while this container runs as root, + # so a module below $GITHUB_WORKSPACE is correctly refused. Keep + # native fixtures in a root-owned, container-ephemeral tree instead. + module_root="/opt/openhuman-test-modules/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + memory_dir="$module_root/tinymemory" + juice_dir="$module_root/tinyjuice" + memory_archive="$memory_dir/tinymemory-module-1.0.1-ubuntu-22.04-x86_64.tar.gz" + juice_archive="$juice_dir/tinyjuice-module-0.2.2-ubuntu-22.04-x86_64.tar.gz" + rm -rf "$memory_dir" "$juice_dir" + mkdir -p "$memory_dir" "$juice_dir" + curl --fail --location --silent --show-error \ + "https://github.com/tinyhumansai/tinymemory/releases/download/v1.0.1/$(basename "$memory_archive")" \ + --output "$memory_archive" + curl --fail --location --silent --show-error \ + "https://github.com/tinyhumansai/tinyjuice/releases/download/v0.2.2/$(basename "$juice_archive")" \ + --output "$juice_archive" + echo "35d13463041f455bebd833a71ba370d891a4e59d1b8e576e81635e2777a0c3dd $memory_archive" \ + | sha256sum --check + echo "fd8caf7fccb53328870fd26922aa9768d253cd4b3bf758967847d6512df03863 $juice_archive" \ + | sha256sum --check + # Do not restore the release builder's uid into the trusted tree. + tar --no-same-owner -xzf "$memory_archive" -C "$memory_dir" + tar --no-same-owner -xzf "$juice_archive" -C "$juice_dir" + echo "TINYMEMORY_TEST_MODULE=$memory_dir/libtinymemory_module.so" \ + >> "$GITHUB_ENV" + echo "TINYJUICE_TEST_MODULE=$juice_dir/libtinyjuice_module.so" \ + >> "$GITHUB_ENV" + + - name: Run TinyJuice host-module regression + run: | + cargo test --lib --features modules \ + openhuman::agent::tinyagents::middleware::tests::tool_output_tabulates_a_large_graph_for_a_non_exempt_tool \ + -- --ignored --exact + env: + CARGO_BUILD_JOBS: "1" + - name: Run cargo llvm-cov for openhuman core (changed modules only) run: bash scripts/ci/rust-coverage-changed.sh env: diff --git a/.github/workflows/release-production.yml b/.github/workflows/release-production.yml index b4be0ba96c..4866ba3f40 100644 --- a/.github/workflows/release-production.yml +++ b/.github/workflows/release-production.yml @@ -401,7 +401,7 @@ jobs: # fork the core image doesn't need. The Dockerfile COPYs vendor/ because # [patch.crates-io] resolves Rust SDK crates from vendor/. - name: Init vendored Rust submodules - run: git submodule update --init --recursive vendor/tinyagents vendor/tinyflows vendor/tinycortex vendor/tinyjuice vendor/tinychannels vendor/tinyplace vendor/tinyhumans-sdk vendor/tinybus vendor/tinymemory + run: git submodule update --init --recursive vendor/tinyagents vendor/tinyflows vendor/tinycortex vendor/tinychannels vendor/tinyplace vendor/tinyhumans-sdk vendor/tinybus vendor/tinymemory - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - name: Log in to GHCR diff --git a/.github/workflows/release-staging.yml b/.github/workflows/release-staging.yml index 0d5bc50de9..842a901e95 100644 --- a/.github/workflows/release-staging.yml +++ b/.github/workflows/release-staging.yml @@ -297,7 +297,7 @@ jobs: # fork the core image doesn't need. The Dockerfile COPYs vendor/ because # [patch.crates-io] resolves Rust SDK crates from vendor/. - name: Init vendored Rust submodules - run: git submodule update --init --recursive vendor/tinyagents vendor/tinyflows vendor/tinycortex vendor/tinyjuice vendor/tinychannels vendor/tinyplace vendor/tinyhumans-sdk vendor/tinybus vendor/tinymemory + run: git submodule update --init --recursive vendor/tinyagents vendor/tinyflows vendor/tinycortex vendor/tinychannels vendor/tinyplace vendor/tinyhumans-sdk vendor/tinybus vendor/tinymemory - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - name: Build image (no push) diff --git a/.gitmodules b/.gitmodules index eccb4cea0f..950fe6ee08 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,9 +7,6 @@ [submodule "vendor/tinycortex"] path = vendor/tinycortex url = https://github.com/tinyhumansai/tinycortex -[submodule "vendor/tinyjuice"] - path = vendor/tinyjuice - url = https://github.com/tinyhumansai/tinyjuice [submodule "vendor/tinychannels"] path = vendor/tinychannels url = https://github.com/tinyhumansai/tinychannels diff --git a/Cargo.lock b/Cargo.lock index 7bc7a556c4..c1dee43948 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1587,15 +1587,6 @@ dependencies = [ "dirs-sys 0.4.1", ] -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys 0.5.0", -] - [[package]] name = "dirs-sys" version = "0.4.1" @@ -4119,7 +4110,7 @@ dependencies = [ "curve25519-dalek", "dhat", "directories", - "dirs 5.0.1", + "dirs", "dotenvy", "ed25519-dalek", "enigo", @@ -4182,7 +4173,6 @@ dependencies = [ "tinycortex-api", "tinyflows", "tinyhumans-sdk", - "tinyjuice", "tinymemory", "tinymemory-api", "tinymemory-core", @@ -6496,7 +6486,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "dirs 5.0.1", + "dirs", "futures", "git2", "hex", @@ -6570,25 +6560,6 @@ dependencies = [ "url", ] -[[package]] -name = "tinyjuice" -version = "0.2.1" -dependencies = [ - "async-trait", - "dirs 6.0.0", - "hex", - "log", - "once_cell", - "regex", - "serde", - "serde_json", - "sha2 0.11.0", - "thiserror 2.0.18", - "tokio", - "unicode-segmentation", - "unicode-width", -] - [[package]] name = "tinymemory" version = "1.0.1" @@ -6625,7 +6596,7 @@ dependencies = [ "async-trait", "axum", "chrono", - "dirs 5.0.1", + "dirs", "futures", "git2", "log", diff --git a/Cargo.toml b/Cargo.toml index b6111a61e9..36b0e367ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,10 +159,6 @@ tinyplace = "2.0" # Optional: exclusive to the default-ON `flows` feature (#4797). A slim build # without `flows` drops this crate and its `jaq-*` JSON-query stack entirely. tinyflows = { version = "0.6", features = ["mock"], optional = true } -# TinyJuice — host-agnostic TokenJuice compression engine. OpenHuman keeps -# config/RPC/tool/runtime adapters in `src/openhuman/tokenjuice/` and patches -# this dependency to the vendored submodule below. -tinyjuice = { version = "0.2.1", default-features = false } # TinyAgents — Rust LLM orchestration framework (LangGraph/LangChain-style): # durable state graphs, agent-loop harness, model/tool registries, REPL + # `.rag` workflow language. openhuman's agent engine + orchestration run on this @@ -1048,7 +1044,7 @@ motosan-ai-oauth = { path = "vendor/motosan-ai-oauth" } # version in lockstep with the `tinyagents` requirement above. After cloning: # `git submodule update --init vendor/tinyagents` (worktrees included). tinyagents = { path = "vendor/tinyagents" } -# TinyFlows, TinyCortex, TinyJuice, TinyChannels, and TinyPlace are vendored beside +# TinyFlows, TinyCortex, TinyChannels, and TinyPlace are vendored beside # TinyAgents so integration work can test crate changes against OpenHuman before # publishing. tinyflows = { path = "vendor/tinyflows" } @@ -1057,7 +1053,6 @@ tinycortex = { path = "vendor/tinycortex" } # depends on it by path, above). Patch it onto the same checkout so both see one # copy of the contract types and the trait identities unify. tinycortex-api = { path = "vendor/tinycortex/api" } -tinyjuice = { path = "vendor/tinyjuice" } tinychannels = { path = "vendor/tinychannels" } tinyplace = { path = "vendor/tinyplace/sdk/rust" } diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 76ad1db14e..3ab00e8f77 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -4725,7 +4725,6 @@ dependencies = [ "tinycortex-api", "tinyflows", "tinyhumans-sdk", - "tinyjuice", "tinymemory", "tinymemory-api", "tinymemory-core", @@ -7704,25 +7703,6 @@ dependencies = [ "url", ] -[[package]] -name = "tinyjuice" -version = "0.2.1" -dependencies = [ - "async-trait", - "dirs 6.0.0", - "hex", - "log", - "once_cell", - "regex", - "serde", - "serde_json", - "sha2 0.11.0", - "thiserror 2.0.20", - "tokio", - "unicode-segmentation", - "unicode-width", -] - [[package]] name = "tinymemory" version = "1.0.1" @@ -8399,12 +8379,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94" -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - [[package]] name = "universal-hash" version = "0.5.1" diff --git a/app/src-tauri/Cargo.toml b/app/src-tauri/Cargo.toml index bbbb4d2d55..ac23b81fb0 100644 --- a/app/src-tauri/Cargo.toml +++ b/app/src-tauri/Cargo.toml @@ -253,7 +253,7 @@ motosan-ai-oauth = { path = "../../vendor/motosan-ai-oauth" } # released tag) — same patch as the root Cargo world so both resolve the # in-tree SDK source. `git submodule update --init vendor/tinyagents` first. tinyagents = { path = "../../vendor/tinyagents" } -# TinyFlows, TinyCortex, TinyJuice, TinyChannels, and TinyPlace are vendored beside +# TinyFlows, TinyCortex, TinyChannels, and TinyPlace are vendored beside # TinyAgents so integration work can test crate changes against OpenHuman before # publishing. tinyflows = { path = "../../vendor/tinyflows" } @@ -263,7 +263,6 @@ tinycortex = { path = "../../vendor/tinycortex" } # patch it onto the same checkout here too, or this independent Cargo world # tries to resolve it from crates.io and fails (it was never published). tinycortex-api = { path = "../../vendor/tinycortex/api" } -tinyjuice = { path = "../../vendor/tinyjuice" } tinychannels = { path = "../../vendor/tinychannels" } tinyplace = { path = "../../vendor/tinyplace/sdk/rust" } diff --git a/scripts/kernel-floor.limits b/scripts/kernel-floor.limits index f7ba939476..b8496066ad 100644 --- a/scripts/kernel-floor.limits +++ b/scripts/kernel-floor.limits @@ -13,6 +13,12 @@ # Simulate with: scripts/dep-sim.py --cut # # History +# 304/281/2 2026-08-14 TinyJuice moved behind the TinyBus module boundary +# (-3 packages / -3 names). The `tinyjuice`, +# `tinyjuice-tokenizer`, and `tinyjuice-vector` +# crates no longer enter the always-on `flows` graph; +# OpenHuman keeps only its stable wire types and host +# adapter. Measured with `scripts/kernel-floor.sh flows`. # 308/284/2 2026-08-13 tinyflows' host-stack advance (PR #5537) moved its # direct HTTP client from reqwest 0.12 to 0.13 while the # rest of the kernel still uses 0.12. That creates one @@ -255,4 +261,4 @@ # (libsqlite3-sys, ring) — see docs/plans MIGRATION-PLAN G6. # 307/284 2026-08-12 Re-baseline after the upstream lockfile resolution; # `flows` remains at two native packages. -flows:308:284:2 +flows:304:281:2 diff --git a/src/core/jsonrpc.rs b/src/core/jsonrpc.rs index 1d9fe94ec8..557f7a5aa7 100644 --- a/src/core/jsonrpc.rs +++ b/src/core/jsonrpc.rs @@ -2058,12 +2058,6 @@ fn register_domain_subscribers( // battery-powered hosts). crate::openhuman::cron::scheduler_gate::init_global(&config); - // Install the TokenJuice content-router runtime config (compressor - // toggles + CCR cache limits + optional on-disk tier). Compaction runs on - // every agent's tool output, so this must be set before any agent loop - // executes a tool. - crate::openhuman::inference::tokenjuice::install_from_config(&config); - // Seed the scheduler-gate signed-out override from the on-disk session. // Without this, a sidecar that boots with no stored JWT would happily // spin up cron / channel loops and fire LLM requests that all 401. diff --git a/src/openhuman/agent/tinyagents/middleware.rs b/src/openhuman/agent/tinyagents/middleware.rs index 12b28ead13..053517bcf5 100644 --- a/src/openhuman/agent/tinyagents/middleware.rs +++ b/src/openhuman/agent/tinyagents/middleware.rs @@ -3654,10 +3654,20 @@ mod tests { } #[tokio::test] + #[ignore = "requires a built TinyJuice module"] async fn tool_output_tabulates_a_large_graph_for_a_non_exempt_tool() { // Sanity baseline proving this test's payload actually exercises real // tinyjuice tabulation (and isn't just below-threshold): a tool name // NOT in COMPACTION_EXEMPT_TOOLS loses the `"type"` marker. + // Resolve the explicit release fixture before `after_tool` performs + // ambient config initialisation. A pristine CI workspace otherwise + // exercises the production fail-open path before the test override is + // admitted, hiding a usable module behind unchanged output. + crate::openhuman::inference::tokenjuice::install_from_config( + &crate::openhuman::config::Config::default(), + ) + .await + .expect("released TinyJuice module must load and accept host configuration"); let mw = compaction_enabled_mw(); let payload = large_workflow_proposal_json(); assert!( diff --git a/src/openhuman/config/schema/tokenjuice.rs b/src/openhuman/config/schema/tokenjuice.rs index 59d286758f..b00c0daca6 100644 --- a/src/openhuman/config/schema/tokenjuice.rs +++ b/src/openhuman/config/schema/tokenjuice.rs @@ -2,8 +2,8 @@ //! //! Controls the TinyJuice content-aware tool-output compaction engine: which //! compressors are enabled, the Compress-Cache-Retrieve (CCR) store limits, and -//! the opt-in Python/ML plain-text compressor. Installed into the runtime at -//! startup via [`crate::openhuman::inference::tokenjuice::install_from_config`]. +//! the opt-in Python/ML plain-text compressor. The host applies these settings +//! before module calls via [`crate::openhuman::inference::tokenjuice::install_from_config`]. use schemars::JsonSchema; use serde::{Deserialize, Serialize}; diff --git a/src/openhuman/inference/tokenjuice/README.md b/src/openhuman/inference/tokenjuice/README.md index f7971fd38f..31d7551a76 100644 --- a/src/openhuman/inference/tokenjuice/README.md +++ b/src/openhuman/inference/tokenjuice/README.md @@ -1,14 +1,15 @@ # OpenHuman TokenJuice Adapter -The reusable compression engine now lives in the vendored `tinyjuice` crate at -`vendor/tinyjuice` and is patched through Cargo. This directory is the -OpenHuman adapter layer. +The reusable compression engine ships as the separately released `tinyjuice` +TinyBus module. It is not linked into OpenHuman's dependency graph. This +directory is the host adapter and shared wire-contract layer. OpenHuman-owned files: | Path | Role | | --- | --- | -| `mod.rs` | Stable OpenHuman module seam, TinyJuice re-exports, config-to-engine install hook, ML/savings callback wiring. | +| `mod.rs` | TinyBus calls, config installation, pass-through fallback, and savings wiring. | +| `types.rs` | Dependency-free copy of the stable JSON wire contract. | | `schemas.rs` | JSON-RPC controller schemas and handlers. | | `config_patch.rs` | Partial update shape for the `[tokenjuice]` config block. | | `tools.rs` | OpenHuman agent tool implementation for `tokenjuice_retrieve`. | @@ -17,14 +18,15 @@ OpenHuman-owned files: TinyJuice-owned engine pieces: -| TinyJuice path | Role | +| TinyJuice repository path | Role | | --- | --- | -| `vendor/tinyjuice/src/compress.rs` | Content router entry point. | -| `vendor/tinyjuice/src/compressors/` | JSON, code, log, search, diff, HTML, ML slot, and generic compressors. | -| `vendor/tinyjuice/src/cache/` | CCR store, retrieval markers, disk tier, ranged retrieval helpers. | -| `vendor/tinyjuice/src/rules/` | Rule loader/compiler and embedded rule table. | -| `vendor/tinyjuice/src/vendor/rules/*.json` | Vendored upstream rule JSON files. | -| `vendor/tinyjuice/src/detect/`, `text/`, `tokens.rs`, `types.rs` | Detection, text helpers, token estimates, public types. | +| `src/compress.rs` | Content router entry point. | +| `src/compressors/` | JSON, code, log, search, diff, HTML, ML slot, and generic compressors. | +| `src/cache/` | CCR store, retrieval markers, disk tier, ranged retrieval helpers. | +| `src/rules/` | Rule loader/compiler and embedded rule table. | +| `src/vendor/rules/*.json` | Vendored upstream rule JSON files. | +| `src/detect/`, `text/`, `tokens.rs`, `types.rs` | Detection, text helpers, token estimates, public types. | -Do not add OpenHuman runtime dependencies to TinyJuice. Runtime services, -settings persistence, JSON-RPC, tools, and pricing stay in this adapter. +Do not add the `tinyjuice` crate back to OpenHuman. Runtime services, settings +persistence, JSON-RPC, tools, pricing, and the optional ML callback stay here; +engine behavior stays behind the loadable module boundary. diff --git a/src/openhuman/inference/tokenjuice/mod.rs b/src/openhuman/inference/tokenjuice/mod.rs index fe10e47e8b..36f6fac3b6 100644 --- a/src/openhuman/inference/tokenjuice/mod.rs +++ b/src/openhuman/inference/tokenjuice/mod.rs @@ -1,56 +1,42 @@ -//! OpenHuman adapter for the vendored TinyJuice compression engine. -//! -//! TinyJuice owns the host-agnostic TokenJuice engine: detection, compressors, -//! CCR cache, rule loading, text helpers, and token estimates. This module keeps -//! the OpenHuman-facing seam stable and owns only host concerns: config mapping, -//! JSON-RPC controllers, settings patching, retrieve tool integration, savings -//! pricing, and the Kompress runtime bridge. - -use std::sync::Arc; +//! OpenHuman host adapter for the separately released TinyJuice module. pub mod config_patch; pub mod ml; pub mod savings; pub mod schemas; pub mod tools; +pub mod types; + +use serde::Serialize; + +pub use tools::TokenjuiceRetrieveTool; +pub use types::{AgentTokenjuiceCompression, CompressorKind, ContentKind}; + +pub const RETRIEVE_TOOL_NAME: &str = "tinyjuice_retrieve"; +pub const LEGACY_RETRIEVE_TOOL_NAME: &str = "retrieve_tool_output"; +pub const RECOVERY_TOOL_NAMES: &[&str] = &[ + RETRIEVE_TOOL_NAME, + "tokenjuice_retrieve", + LEGACY_RETRIEVE_TOOL_NAME, +]; -pub use tinyjuice::{ - cache, classify, compress, compressors, detect, reduce, rules, text, tokens, tool_integration, - types, -}; - -/// Install the full TokenJuice runtime from a [`Config`] in one call: router / -/// compressor options + CCR cache limits + disk tier, savings attribution + -/// snapshot path, and the ML backend config snapshot. Used at startup and after -/// a live settings update. -/// -/// Note: toggling `ml_compression_enabled` and the live compressor/CCR flags -/// takes effect immediately; the ML model id / device snapshot is read once and -/// only changes on restart. -pub fn install_from_config(config: &crate::openhuman::config::Config) { +pub fn is_recovery_tool(name: &str) -> bool { + RECOVERY_TOOL_NAMES.contains(&name) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct InstallRequest { + options: types::CompressOptions, + max_cache_entries: usize, + max_cache_bytes: usize, + ccr_ttl_secs: Option, + disk_tier_root: Option, +} + +pub async fn install_from_config(config: &crate::openhuman::config::Config) -> Result<(), String> { let tj = &config.tokenjuice; - let options = tinyjuice::types::CompressOptions { - router_enabled: tj.router_enabled, - ccr_enabled: tj.ccr_enabled, - search_enabled: tj.search_enabled, - code_enabled: tj.code_enabled, - html_enabled: tj.html_enabled, - ml_text_enabled: tj.ml_compression_enabled, - min_bytes_to_compress: tj.min_bytes_to_compress, - ccr_min_tokens: tj.ccr_min_tokens, - max_inline_chars: None, - ..Default::default() - }; - let disk_root = tj - .ccr_disk_enabled - .then(|| config.workspace_dir.join(".tokenjuice").join("ccr")); - tinyjuice::tool_integration::install_config( - options, - tj.max_cache_entries, - tj.max_cache_bytes, - tj.ccr_ttl_secs, - disk_root, - ); + ml::configure(config.clone()); savings::configure( config .default_model @@ -58,49 +44,252 @@ pub fn install_from_config(config: &crate::openhuman::config::Config) { .unwrap_or_else(|| crate::openhuman::config::DEFAULT_MODEL.to_string()), &config.workspace_dir, ); - tinyjuice::savings::configure_recorder(Some(Arc::new( - |content_kind, compressor, original_tokens, compacted_tokens| { - savings::record(content_kind, compressor, original_tokens, compacted_tokens); - }, - ))); - ml::configure(config.clone()); - tinyjuice::ml::configure_callback(Some(Arc::new( - |text: String, opts: tinyjuice::types::CompressOptions| { - Box::pin(async move { - ml::compress(&text, &opts) - .await - .map_err(|err| format!("{err:#}")) - }) + let request = InstallRequest { + options: types::CompressOptions { + router_enabled: tj.router_enabled, + ccr_enabled: tj.ccr_enabled, + search_enabled: tj.search_enabled, + code_enabled: tj.code_enabled, + html_enabled: tj.html_enabled, + ml_text_enabled: tj.ml_compression_enabled, + min_bytes_to_compress: tj.min_bytes_to_compress, + ccr_min_tokens: tj.ccr_min_tokens, + ..types::CompressOptions::default() }, - ))); + max_cache_entries: tj.max_cache_entries, + max_cache_bytes: tj.max_cache_bytes, + ccr_ttl_secs: tj.ccr_ttl_secs, + disk_tier_root: tj + .ccr_disk_enabled + .then(|| config.workspace_dir.join(".tokenjuice").join("ccr")) + .map(|path| path.to_string_lossy().into_owned()), + }; + let fingerprint = serde_json::to_vec(&request).map_err(|error| error.to_string())?; + static INSTALLED: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + let mut installed = INSTALLED + .get_or_init(|| tokio::sync::Mutex::new(None)) + .lock() + .await; + if installed.as_ref() == Some(&fingerprint) { + return Ok(()); + } + proxy(config) + .await? + .call::<()>("Install", (request,)) + .await + .map_err(|e| e.to_string())?; + *installed = Some(fingerprint); + Ok(()) +} + +#[cfg(feature = "modules")] +pub(super) async fn proxy( + config: &crate::openhuman::config::Config, +) -> Result { + #[cfg(test)] + let config = { + let mut test_config = config.clone(); + if let Some(path) = std::env::var_os("TINYJUICE_TEST_MODULE") { + // An explicit fixture is an opt-in to module execution even when + // the ambient test workspace has persisted modules = disabled. + test_config.modules.enabled = true; + test_config + .modules + .overrides + .push(crate::openhuman::config::schema::ModuleOverride { + id: "tinyjuice".to_string(), + path: path.to_string_lossy().into_owned(), + }); + } + test_config + }; + #[cfg(test)] + let config = &config; + + crate::openhuman::modules::ensure_loaded(config, "tinyjuice").await?; + let record = crate::openhuman::modules::registry::find("tinyjuice") + .ok_or_else(|| "unknown module 'tinyjuice'".to_string())?; + crate::openhuman::modules::host::runtime() + .await + .map_err(|e| e.to_string())? + .proxy(record.bus_name, record.object_path) + .map_err(|e| e.to_string()) +} + +#[cfg(not(feature = "modules"))] +pub(super) async fn proxy( + _config: &crate::openhuman::config::Config, +) -> Result { + Err("native modules are not compiled into this build".to_string()) +} + +pub async fn compact_output_with_policy( + content: String, + tool_name: &str, + enabled: bool, + profile: AgentTokenjuiceCompression, +) -> String { + if !enabled || profile == AgentTokenjuiceCompression::Off { + return content; + } + let config = match crate::openhuman::config::Config::load_or_init().await { + Ok(config) => config, + Err(error) => { + log::debug!("[tokenjuice] config unavailable, passing through: {error}"); + return content; + } + }; + #[cfg(test)] + let config = if std::env::var_os("TINYJUICE_TEST_MODULE").is_some() { + // The released-module regression must not inherit an operator's + // persisted compression thresholds or disabled router flags. + crate::openhuman::config::Config::default() + } else { + config + }; + if let Err(error) = install_from_config(&config).await { + log::debug!("[tokenjuice] module configuration failed, passing through: {error}"); + return content; + } + let proxy = match proxy(&config).await { + Ok(proxy) => proxy, + Err(error) => { + log::debug!("[tokenjuice] module unavailable, passing through: {error}"); + return content; + } + }; + let response: types::CompactResponse = match proxy + .call( + "Compact", + (content.clone(), tool_name.to_string(), enabled, profile), + ) + .await + { + Ok(response) => response, + Err(error) => { + log::debug!("[tokenjuice] module compaction failed, passing through: {error}"); + return content; + } + }; + record_savings(&response); + response.text +} + +fn record_savings(response: &types::CompactResponse) { + use std::str::FromStr as _; + let kind = ContentKind::from_str(&response.content_kind).unwrap_or(ContentKind::PlainText); + let compressor = CompressorKind::from_str(&response.compressor).unwrap_or(CompressorKind::None); + savings::record( + kind, + compressor, + response.original_tokens, + response.compacted_tokens, + ); +} + +pub async fn detect(content: String, hint: types::ContentHint) -> Result { + let config = crate::openhuman::config::Config::load_or_init() + .await + .map_err(|error| error.to_string())?; + proxy(&config) + .await? + .call("Detect", (content, hint)) + .await + .map_err(|error| error.to_string()) +} + +pub async fn compress( + content: String, + hint: types::ContentHint, +) -> Result { + let config = crate::openhuman::config::Config::load_or_init() + .await + .map_err(|error| error.to_string())?; + install_from_config(&config).await?; + let response: types::CompressedOutput = proxy(&config) + .await? + .call("Compress", (content, hint)) + .await + .map_err(|error| error.to_string())?; + savings::record( + response.content_kind, + response.compressor, + (response.original_bytes as u64).div_ceil(4), + (response.compacted_bytes as u64).div_ceil(4), + ); + Ok(response) +} + +pub async fn retrieve( + token: String, + range: Option, +) -> Result, String> { + let config = crate::openhuman::config::Config::load_or_init() + .await + .map_err(|error| error.to_string())?; + install_from_config(&config).await?; + proxy(&config) + .await? + .call("Retrieve", (token, range)) + .await + .map_err(|error| error.to_string()) +} + +pub async fn cache_stats() -> Result { + let config = crate::openhuman::config::Config::load_or_init() + .await + .map_err(|error| error.to_string())?; + install_from_config(&config).await?; + proxy(&config) + .await? + .call("CacheStats", ()) + .await + .map_err(|error| error.to_string()) } -/// All read-only TokenJuice debug controllers (detect / compress / cache_stats -/// / retrieve), for registration in `src/core/all.rs`. pub fn all_tokenjuice_registered_controllers() -> Vec { schemas::all_registered_controllers() } -/// Declared schemas for the TokenJuice debug controllers. pub fn all_tokenjuice_controller_schemas() -> Vec { schemas::all_controller_schemas() } -pub use cache::{ - is_recovery_tool, LEGACY_RETRIEVE_TOOL_NAME, NEVER_COMPACT_TOOLS, RECOVERY_TOOL_NAMES, - RETRIEVE_TOOL_NAME, -}; -pub use compress::{compress_content, route}; -pub use compressors::{compressor_for, generic_compressor, Compressor}; -pub use detect::detect_content_kind; -pub use reduce::reduce_execution_with_rules; -pub use rules::{load_builtin_rules, load_rules, LoadRuleOptions}; -pub use tool_integration::{ - compact_output, compact_output_with_policy, compact_tool_output_with_policy, configure, - current_options, install_config, CompactionStats, -}; -pub use tools::TokenjuiceRetrieveTool; -pub use types::{ - AgentTokenjuiceCompression, CompactResult, CompressInput, CompressOptions, CompressOutput, - CompressedOutput, CompressorKind, ContentHint, ContentKind, ReduceOptions, ToolExecutionInput, -}; +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recovery_tool_aliases_remain_stable() { + assert!(is_recovery_tool(RETRIEVE_TOOL_NAME)); + assert!(is_recovery_tool(LEGACY_RETRIEVE_TOOL_NAME)); + assert!(!is_recovery_tool("shell")); + } + + #[tokio::test] + async fn disabled_compaction_is_an_exact_pass_through_without_loading_the_module() { + let content = "exact tool output".to_string(); + let output = compact_output_with_policy( + content.clone(), + "shell", + false, + AgentTokenjuiceCompression::Full, + ) + .await; + assert_eq!(output, content); + } + + #[tokio::test] + async fn off_profile_is_an_exact_pass_through_without_loading_the_module() { + let content = "exact tool output".to_string(); + let output = compact_output_with_policy( + content.clone(), + "shell", + true, + AgentTokenjuiceCompression::Off, + ) + .await; + assert_eq!(output, content); + } +} diff --git a/src/openhuman/inference/tokenjuice/savings.rs b/src/openhuman/inference/tokenjuice/savings.rs index 9f2d2f676c..6e11fa5bba 100644 --- a/src/openhuman/inference/tokenjuice/savings.rs +++ b/src/openhuman/inference/tokenjuice/savings.rs @@ -126,17 +126,24 @@ fn resolve_attribution_model(default: &str) -> String { .unwrap_or_else(|| default.to_string()) } -/// Install the attribution model and snapshot location, loading any prior -/// snapshot. Called once at startup from [`crate::openhuman::inference::tokenjuice::install_config`]. +/// Install the attribution model and snapshot location, loading a prior +/// snapshot once per workspace. pub fn configure(attribution_model: String, workspace_dir: &std::path::Path) { let path = workspace_dir.join("state").join("tokenjuice_savings.json"); - let loaded = std::fs::read_to_string(&path) - .ok() - .and_then(|s| serde_json::from_str::(&s).ok()); let mut st = state().lock().unwrap_or_else(|p| p.into_inner()); if !attribution_model.trim().is_empty() { st.attribution_model = attribution_model; } + // Module calls apply host configuration lazily. Do not reload the snapshot + // on every tool result: besides needless I/O, a concurrent call could have + // read the file just before another call persisted a saving and then replace + // the newer in-memory aggregate with that stale copy. + if st.snapshot_path.as_ref() == Some(&path) { + return; + } + let loaded = std::fs::read_to_string(&path) + .ok() + .and_then(|s| serde_json::from_str::(&s).ok()); st.snapshot_path = Some(path); if let Some(agg) = loaded { st.aggregate = agg; diff --git a/src/openhuman/inference/tokenjuice/schemas.rs b/src/openhuman/inference/tokenjuice/schemas.rs index 699608fbdf..68bec64f9f 100644 --- a/src/openhuman/inference/tokenjuice/schemas.rs +++ b/src/openhuman/inference/tokenjuice/schemas.rs @@ -11,11 +11,7 @@ use serde_json::{Map, Value}; use crate::core::all::{ControllerFuture, RegisteredController}; use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; -use super::cache; -use super::compress::route; -use super::detect::detect_content_kind; -use super::tool_integration::current_options; -use super::types::{CompressInput, ContentHint}; +use super::types::ContentHint; pub fn all_controller_schemas() -> Vec { vec![ @@ -237,8 +233,8 @@ fn handle_detect(params: Map) -> ControllerFuture { extension: str_param(¶ms, "extension"), ..Default::default() }; - let kind = detect_content_kind(&content, &hint); - Ok(serde_json::json!({ "kind": kind.as_str() })) + let kind = super::detect(content, hint).await?; + Ok(serde_json::json!({ "kind": kind })) }) } @@ -249,17 +245,7 @@ fn handle_compress(params: Map) -> ControllerFuture { source_tool: str_param(¶ms, "tool_name"), ..Default::default() }; - let opts = current_options(); - let input = CompressInput { - content: &content, - kind: super::types::ContentKind::PlainText, - hint: &hint, - exit_code: None, - command: None, - argv: None, - original_bytes: content.len(), - }; - let res = route(input, &opts).await; + let res = super::compress(content, hint).await?; Ok(serde_json::json!({ "applied": res.applied, "kind": res.content_kind.as_str(), @@ -275,15 +261,15 @@ fn handle_compress(params: Map) -> ControllerFuture { fn handle_cache_stats(_params: Map) -> ControllerFuture { Box::pin(async move { - let (entries, bytes) = cache::stats(); - Ok(serde_json::json!({ "entries": entries, "bytes": bytes })) + let stats = super::cache_stats().await?; + Ok(serde_json::json!({ "entries": stats.entries, "bytes": stats.bytes })) }) } fn handle_retrieve(params: Map) -> ControllerFuture { Box::pin(async move { let token = str_param(¶ms, "token").ok_or("missing 'token'")?; - match cache::retrieve(&token) { + match super::retrieve(token, None).await? { Some(content) => Ok(serde_json::json!({ "found": true, "content": content })), None => Ok(serde_json::json!({ "found": false, "content": Value::Null })), } @@ -321,7 +307,7 @@ fn handle_settings_update(params: Map) -> ControllerFuture { .map_err(|e| format!("save config: {e}"))?; // Re-install so router flags / CCR limits / threshold take effect live. - crate::openhuman::inference::tokenjuice::install_from_config(&config); + crate::openhuman::inference::tokenjuice::install_from_config(&config).await?; let settings = serde_json::to_value(&config.tokenjuice) .map_err(|e| format!("serialize tokenjuice settings: {e}"))?; @@ -331,14 +317,14 @@ fn handle_settings_update(params: Map) -> ControllerFuture { fn handle_savings_stats(_params: Map) -> ControllerFuture { Box::pin(async move { + let cache = super::cache_stats().await?; let agg = super::savings::stats(); - let (entries, bytes) = cache::stats(); Ok(serde_json::json!({ "attributionModel": super::savings::attribution_model(), "total": agg.total, "byModel": agg.by_model, "byCompressor": agg.by_compressor, - "cache": { "entries": entries, "bytes": bytes }, + "cache": { "entries": cache.entries, "bytes": cache.bytes }, })) }) } @@ -354,7 +340,16 @@ fn handle_savings_reset(_params: Map) -> ControllerFuture { mod tests { use super::*; + #[test] + fn all_schemas_have_namespace() { + for s in all_controller_schemas() { + assert_eq!(s.namespace, "tokenjuice"); + } + } + + /* Module-backed behavior is covered by TinyJuice's loader E2E. */ #[tokio::test] + #[ignore = "requires a built TinyJuice module"] async fn detect_handler_classifies_json() { let mut p = Map::new(); p.insert( @@ -366,16 +361,9 @@ mod tests { } #[tokio::test] + #[ignore = "requires a built TinyJuice module"] async fn cache_stats_handler_returns_counts() { - cache::offload("tokenjuice controller stats unique payload here"); let out = handle_cache_stats(Map::new()).await.unwrap(); - assert!(out["entries"].as_u64().unwrap() >= 1); - } - - #[test] - fn all_schemas_have_namespace() { - for s in all_controller_schemas() { - assert_eq!(s.namespace, "tokenjuice"); - } + assert!(out["entries"].is_u64()); } } diff --git a/src/openhuman/inference/tokenjuice/tools.rs b/src/openhuman/inference/tokenjuice/tools.rs index ca8a1381fd..e86ff0c068 100644 --- a/src/openhuman/inference/tokenjuice/tools.rs +++ b/src/openhuman/inference/tokenjuice/tools.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use serde_json::{json, Value}; -use crate::openhuman::inference::tokenjuice::cache::{self, store::RangeUnit}; +use crate::openhuman::inference::tokenjuice::types::{RangeUnit, RetrieveRange}; use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; pub struct TokenjuiceRetrieveTool; @@ -31,7 +31,7 @@ impl Default for TokenjuiceRetrieveTool { #[async_trait] impl Tool for TokenjuiceRetrieveTool { fn name(&self) -> &str { - cache::RETRIEVE_TOOL_NAME + super::RETRIEVE_TOOL_NAME } fn description(&self) -> &str { @@ -92,27 +92,34 @@ impl Tool for TokenjuiceRetrieveTool { Some("bytes") => RangeUnit::Bytes, _ => RangeUnit::Lines, }; - return match cache::retrieve_range(token, start, end, unit) { - Some(slice) => { + return match super::retrieve( + token.to_string(), + Some(RetrieveRange { start, end, unit }), + ) + .await + { + Ok(Some(slice)) => { log::debug!( "[tokenjuice][ccr] retrieved range token={token} {start}..{end} {} bytes", slice.len() ); Ok(ToolResult::success(slice)) } - None => Ok(ToolResult::error(miss_message(token))), + Ok(None) => Ok(ToolResult::error(miss_message(token))), + Err(error) => Ok(ToolResult::error(error)), }; } - match cache::retrieve(token) { - Some(original) => { + match super::retrieve(token.to_string(), None).await { + Ok(Some(original)) => { log::debug!( "[tokenjuice][ccr] retrieved token={token} bytes={}", original.len() ); Ok(ToolResult::success(original)) } - None => Ok(ToolResult::error(miss_message(token))), + Ok(None) => Ok(ToolResult::error(miss_message(token))), + Err(error) => Ok(ToolResult::error(error)), } } } @@ -127,12 +134,11 @@ fn miss_message(token: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::inference::tokenjuice::cache::store; - #[tokio::test] + #[ignore = "requires a built TinyJuice module"] async fn retrieves_offloaded_original() { let original = "ORIGINAL TOKENJUICE PAYLOAD ".repeat(20); - let hash = store::offload(&original); + let hash = "module-fixture"; let tool = TokenjuiceRetrieveTool::new(); let res = tool.execute(json!({ "token": hash })).await.unwrap(); assert!(!res.is_error); @@ -140,9 +146,10 @@ mod tests { } #[tokio::test] + #[ignore = "requires a built TinyJuice module"] async fn retrieves_line_range() { - let original = "r0\nr1\nr2\nr3\nr4"; - let hash = store::offload(original); + let _original = "r0\nr1\nr2\nr3\nr4"; + let hash = "module-fixture"; let tool = TokenjuiceRetrieveTool::new(); let res = tool .execute(json!({ "token": hash, "range": { "start": 1, "end": 3, "unit": "lines" } })) diff --git a/src/openhuman/inference/tokenjuice/types.rs b/src/openhuman/inference/tokenjuice/types.rs new file mode 100644 index 0000000000..89b143731f --- /dev/null +++ b/src/openhuman/inference/tokenjuice/types.rs @@ -0,0 +1,259 @@ +//! Stable wire types shared with the separately compiled TinyJuice module. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum AgentTokenjuiceCompression { + #[default] + Auto, + Full, + Light, + Off, +} + +impl AgentTokenjuiceCompression { + pub fn as_str(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::Full => "full", + Self::Light => "light", + Self::Off => "off", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ContentKind { + Json, + Code, + Log, + Search, + Diff, + Html, + PlainText, +} + +impl ContentKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Json => "json", + Self::Code => "code", + Self::Log => "log", + Self::Search => "search", + Self::Diff => "diff", + Self::Html => "html", + Self::PlainText => "plain_text", + } + } +} + +impl std::str::FromStr for ContentKind { + type Err = (); + fn from_str(value: &str) -> Result { + Ok(match value { + "json" => Self::Json, + "code" => Self::Code, + "log" => Self::Log, + "search" => Self::Search, + "diff" => Self::Diff, + "html" => Self::Html, + "plain_text" => Self::PlainText, + _ => return Err(()), + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum CompressorKind { + SmartCrusher, + Code, + Log, + Search, + Diff, + Html, + MlText, + Generic, + None, +} + +impl CompressorKind { + pub fn as_str(self) -> &'static str { + match self { + Self::SmartCrusher => "smartcrusher", + Self::Code => "code", + Self::Log => "log", + Self::Search => "search", + Self::Diff => "diff", + Self::Html => "html", + Self::MlText => "ml_text", + Self::Generic => "generic", + Self::None => "none", + } + } +} + +impl std::str::FromStr for CompressorKind { + type Err = (); + fn from_str(value: &str) -> Result { + Ok(match value { + "smartcrusher" => Self::SmartCrusher, + "code" => Self::Code, + "log" => Self::Log, + "search" => Self::Search, + "diff" => Self::Diff, + "html" => Self::Html, + "ml_text" => Self::MlText, + "generic" => Self::Generic, + "none" => Self::None, + _ => return Err(()), + }) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContentHint { + pub mime: Option, + pub extension: Option, + pub source_tool: Option, + pub query: Option, + pub explicit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CompressOptions { + pub router_enabled: bool, + pub ccr_enabled: bool, + pub search_enabled: bool, + pub code_enabled: bool, + pub html_enabled: bool, + pub ml_text_enabled: bool, + pub min_bytes_to_compress: usize, + pub min_bytes_to_compress_log: usize, + pub ccr_min_tokens: usize, + pub lossy_without_ccr: bool, + pub max_inline_chars: Option, + pub code_target_ratio: Option, + pub chars_per_token: f32, +} + +impl Default for CompressOptions { + fn default() -> Self { + Self { + router_enabled: true, + ccr_enabled: true, + search_enabled: true, + code_enabled: true, + html_enabled: true, + ml_text_enabled: false, + min_bytes_to_compress: 2048, + min_bytes_to_compress_log: 512, + ccr_min_tokens: 500, + lossy_without_ccr: false, + max_inline_chars: None, + code_target_ratio: None, + chars_per_token: 4.0, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompressedOutput { + pub text: String, + pub content_kind: ContentKind, + pub compressor: CompressorKind, + pub lossy: bool, + pub applied: bool, + pub ccr_token: Option, + pub original_bytes: usize, + pub compacted_bytes: usize, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompactResponse { + pub text: String, + pub original_bytes: usize, + pub compacted_bytes: usize, + pub rule_id: String, + pub applied: bool, + pub content_kind: String, + pub compressor: String, + pub original_tokens: u64, + pub compacted_tokens: u64, +} + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum RangeUnit { + Bytes, + Lines, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RetrieveRange { + pub start: usize, + pub end: usize, + pub unit: RangeUnit, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CacheStats { + pub entries: usize, + pub bytes: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compress_options_accept_omitted_fields() { + let options: CompressOptions = serde_json::from_value(serde_json::json!({ + "routerEnabled": false + })) + .expect("partial options remain forward-compatible"); + assert!(!options.router_enabled); + assert!(options.ccr_enabled); + assert_eq!(options.min_bytes_to_compress, 2048); + } + + #[test] + fn content_hint_matches_the_module_wire_shape() { + let hint = ContentHint { + source_tool: Some("shell".to_string()), + explicit: Some(ContentKind::PlainText), + ..ContentHint::default() + }; + assert_eq!( + serde_json::to_value(hint).expect("serialize hint"), + serde_json::json!({ + "mime": null, + "extension": null, + "sourceTool": "shell", + "query": null, + "explicit": "plainText" + }) + ); + } + + #[test] + fn retrieve_range_uses_camel_case_wire_values() { + let range = RetrieveRange { + start: 2, + end: 5, + unit: RangeUnit::Lines, + }; + assert_eq!( + serde_json::to_value(range).expect("serialize range"), + serde_json::json!({ "start": 2, "end": 5, "unit": "lines" }) + ); + } +} diff --git a/src/openhuman/modules/host.rs b/src/openhuman/modules/host.rs index 9880fcd410..fee9649468 100644 --- a/src/openhuman/modules/host.rs +++ b/src/openhuman/modules/host.rs @@ -177,6 +177,7 @@ async fn build_runtime() -> tinybus::Result { if let Some(config) = super::memory::policy().cloned() { super::memory_host::install(&connection, Arc::clone(&config)).await?; } + super::tokenjuice_host::install(&connection).await?; Ok(ModuleRuntime { host, diff --git a/src/openhuman/modules/memory.rs b/src/openhuman/modules/memory.rs index ddcbd8655a..cf6b63cd9b 100644 --- a/src/openhuman/modules/memory.rs +++ b/src/openhuman/modules/memory.rs @@ -177,12 +177,6 @@ impl ModuleMemoryProvider { cannot be loaded; call modules::memory::set_modules_policy during boot" )) })?; - ops::ensure_loaded(config, MODULE_ID) - .await - .map_err(|message| MemoryError::Other(anyhow::anyhow!(message)))?; - - let record = registry::find(MODULE_ID) - .ok_or_else(|| MemoryError::Other(anyhow::anyhow!("unknown module '{MODULE_ID}'")))?; let runtime = host::runtime().await.map_err(|error| { MemoryError::Other(anyhow::anyhow!("the module bus is not running: {error}")) })?; @@ -193,6 +187,16 @@ impl ModuleMemoryProvider { "the memory module host callbacks are unavailable: {error}" )) })?; + // TinyMemory resolves its embedding provider while the native library + // is admitted. Host callbacks must therefore exist before loading, + // including in tests and explicit-path overrides where no boot policy + // was available when the shared module runtime first started. + ops::ensure_loaded(config, MODULE_ID) + .await + .map_err(|message| MemoryError::Other(anyhow::anyhow!(message)))?; + + let record = registry::find(MODULE_ID) + .ok_or_else(|| MemoryError::Other(anyhow::anyhow!("unknown module '{MODULE_ID}'")))?; let proxy = runtime .proxy(record.bus_name, record.object_path) .map_err(|error| MemoryError::Other(anyhow::anyhow!(error.to_string())))?; diff --git a/src/openhuman/modules/mod.rs b/src/openhuman/modules/mod.rs index abb7e5b013..dc61bdf4a5 100644 --- a/src/openhuman/modules/mod.rs +++ b/src/openhuman/modules/mod.rs @@ -46,6 +46,7 @@ pub mod ops; pub mod platform; pub mod registry; pub mod schemas; +mod tokenjuice_host; pub mod types; #[cfg(feature = "web3")] pub mod wallet; diff --git a/src/openhuman/modules/registry.rs b/src/openhuman/modules/registry.rs index cfea6b04b1..58374ac6bf 100644 --- a/src/openhuman/modules/registry.rs +++ b/src/openhuman/modules/registry.rs @@ -248,8 +248,80 @@ const TINYMEMORY: ModuleRecord = ModuleRecord { load: LoadPolicy::Eager, }; +/// The `tinyjuice` content-aware tool-output compression engine. +/// +/// Lazy because the host's compaction policy can disable it, and a session that +/// never produces compressible tool output should not pay the download or +/// resident native-library cost. +const TINYJUICE: ModuleRecord = ModuleRecord { + id: "tinyjuice", + description: "Content-aware tool-output compression and recoverable caching", + bus_name: "ai.tinyhumans.tinyjuice.Compression", + object_path: "/ai/tinyhumans/tinyjuice/Compression", + version: "0.2.2", + release_url: "https://github.com/tinyhumansai/tinyjuice/releases/tag/v0.2.2", + assets: &[ + PlatformAsset { + host_key: "ubuntu-24.04-x86_64", + archive: "tinyjuice-module-0.2.2-ubuntu-24.04-x86_64.tar.gz", + sha256: "ed80892f82e9ba824bb1cc436adf2ad77bc4ba59205a3bdb1eecd96841797a16", + }, + PlatformAsset { + host_key: "ubuntu-24.04-arm64", + archive: "tinyjuice-module-0.2.2-ubuntu-24.04-arm64.tar.gz", + sha256: "91b16e77671c0c06ca3c413bddc7218b6d65453eb7b43d87d58b693fd8273a55", + }, + PlatformAsset { + host_key: "ubuntu-22.04-x86_64", + archive: "tinyjuice-module-0.2.2-ubuntu-22.04-x86_64.tar.gz", + sha256: "fd8caf7fccb53328870fd26922aa9768d253cd4b3bf758967847d6512df03863", + }, + PlatformAsset { + host_key: "ubuntu-22.04-arm64", + archive: "tinyjuice-module-0.2.2-ubuntu-22.04-arm64.tar.gz", + sha256: "10e70614aca9da5d108c7335b73238e81de3e9daaad8291a690ef5d2bb48e852", + }, + PlatformAsset { + host_key: "macos-26-arm64", + archive: "tinyjuice-module-0.2.2-macos-26-arm64.tar.gz", + sha256: "30dc34f2901e1581f72c1d718b80632268714193964031ad52151dd6f046b5b8", + }, + PlatformAsset { + host_key: "macos-26-x86_64", + archive: "tinyjuice-module-0.2.2-macos-26-x86_64.tar.gz", + sha256: "122bac614bb2d27717b0ce5d0661b1ee10810b2e3c3417f153daa7a783f706a9", + }, + PlatformAsset { + host_key: "macos-15-arm64", + archive: "tinyjuice-module-0.2.2-macos-15-arm64.tar.gz", + sha256: "cf833e0315ecab66a6fd99695065745f04b1ceb5169d2e7d3227b9ff60828a0c", + }, + PlatformAsset { + host_key: "macos-15-x86_64", + archive: "tinyjuice-module-0.2.2-macos-15-x86_64.tar.gz", + sha256: "ce28e5c4e06dab98b376defd09d2c4f7fd85b235c0daae1a9bd5e941c8085833", + }, + PlatformAsset { + host_key: "windows-2025-x86_64", + archive: "tinyjuice-module-0.2.2-windows-2025-x86_64.zip", + sha256: "b22df6573abf7376252ce3f62e339870719dfceee9d8bfc0752b7f1cdd92ded0", + }, + PlatformAsset { + host_key: "windows-2022-x86_64", + archive: "tinyjuice-module-0.2.2-windows-2022-x86_64.zip", + sha256: "dc44e589fc50b2d5e33d493a2547e38db7e7e9a28012c616b3155db2ff15c5cf", + }, + PlatformAsset { + host_key: "windows-11-arm64", + archive: "tinyjuice-module-0.2.2-windows-11-arm64.zip", + sha256: "0b9389abae5f3432a02f0c18bfea33187e7cc2634a12281f2bdb67bb5501e338", + }, + ], + load: LoadPolicy::Lazy, +}; + /// Every module this build can load. -pub const ALL: &[ModuleRecord] = &[TINYDOCS, TINYWALLET, TINYMEMORY]; +pub const ALL: &[ModuleRecord] = &[TINYDOCS, TINYWALLET, TINYMEMORY, TINYJUICE]; /// The record for `id`, if this build knows it. #[must_use] diff --git a/src/openhuman/modules/tokenjuice_host.rs b/src/openhuman/modules/tokenjuice_host.rs new file mode 100644 index 0000000000..32fc93f60f --- /dev/null +++ b/src/openhuman/modules/tokenjuice_host.rs @@ -0,0 +1,35 @@ +//! Host-owned optional ML callback served to the TinyJuice module. + +use tinybus::ObjectPath; + +const NAME: &str = "ai.tinyhumans.tinyjuice.MlHost"; +const PATH: &str = "/ai/tinyhumans/tinyjuice/MlHost"; + +#[derive(Clone)] +struct MlHost; + +#[tinybus::interface(name = "ai.tinyhumans.tinyjuice.MlHost")] +impl MlHost { + async fn compress( + &self, + text: String, + options: serde_json::Value, + ) -> tinybus::Result> { + let options = serde_json::from_value(options).map_err(method_error)?; + crate::openhuman::inference::tokenjuice::ml::compress(&text, &options) + .await + .map_err(method_error) + } +} + +fn method_error(error: impl std::fmt::Display) -> tinybus::Error { + tinybus::Error::MethodFailed { + name: "ai.tinyhumans.tinyjuice.Error.Host".to_string(), + message: error.to_string(), + } +} + +pub(super) async fn install(connection: &tinybus::Connection) -> tinybus::Result<()> { + connection.serve_at(ObjectPath::new(PATH)?, MlHost).await?; + connection.request_name(NAME).await +} diff --git a/src/openhuman/tools/impl/system/retrieve_tool_output.rs b/src/openhuman/tools/impl/system/retrieve_tool_output.rs index 6e7c91d08b..948455a1ee 100644 --- a/src/openhuman/tools/impl/system/retrieve_tool_output.rs +++ b/src/openhuman/tools/impl/system/retrieve_tool_output.rs @@ -68,8 +68,8 @@ impl Tool for RetrieveToolOutputTool { )); }; - match crate::openhuman::inference::tokenjuice::cache::retrieve(hash) { - Some(original) => { + match crate::openhuman::inference::tokenjuice::retrieve(hash.to_string(), None).await { + Ok(Some(original)) => { log::debug!( "[compaction][ccr] retrieved hash={} bytes={}", hash, @@ -77,10 +77,11 @@ impl Tool for RetrieveToolOutputTool { ); Ok(ToolResult::success(original)) } - None => Ok(ToolResult::error(format!( + Ok(None) => Ok(ToolResult::error(format!( "retrieve_tool_output: no cached original for hash '{hash}' \ (it may have been evicted; re-run the tool to regenerate it)" ))), + Err(error) => Ok(ToolResult::error(error)), } } } @@ -88,12 +89,11 @@ impl Tool for RetrieveToolOutputTool { #[cfg(test)] mod tests { use super::*; - use crate::openhuman::inference::tokenjuice::cache::store; - #[tokio::test] + #[ignore = "requires a built TinyJuice module"] async fn retrieves_offloaded_original() { let original = "ORIGINAL PAYLOAD ".repeat(20); - let hash = store::offload(&original); + let hash = "module-fixture"; let tool = RetrieveToolOutputTool::new(); let res = tool.execute(json!({ "hash": hash })).await.unwrap(); assert!(!res.is_error); diff --git a/tests/tokenjuice_integration.rs b/tests/tokenjuice_integration.rs deleted file mode 100644 index 5ed2ea8fe1..0000000000 --- a/tests/tokenjuice_integration.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! Integration tests for the TokenJuice module. -//! -//! Iterates vendored TinyJuice `*.fixture.json` files under -//! `vendor/tinyjuice/tests/fixtures/` and asserts that -//! `reduce_execution_with_rules` produces the expected output. - -use openhuman_core::openhuman::inference::tokenjuice::{ - reduce::reduce_execution_with_rules, rules::load_builtin_rules, types::RuleFixture, -}; - -/// Fixture names that are known to produce different output from the upstream -/// TypeScript — typically due to `Intl.Segmenter` vs `unicode-segmentation` -/// grapheme-boundary differences. See `KNOWN_DRIFT.md` for rationale. -const KNOWN_DRIFT_FIXTURES: &[&str] = &[ - // None currently. -]; - -fn fixtures_dir() -> std::path::PathBuf { - let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"); - std::path::PathBuf::from(manifest).join("vendor/tinyjuice/tests/fixtures") -} - -#[test] -fn vendored_fixtures_match_expected_output() { - let dir = fixtures_dir(); - assert!( - dir.is_dir(), - "fixtures directory not found: {}", - dir.display() - ); - - let rules = load_builtin_rules(); - let mut entries: Vec<_> = std::fs::read_dir(&dir) - .expect("read fixtures dir") - .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().ends_with(".fixture.json")) - .collect(); - entries.sort_by_key(|e| e.file_name()); - - let mut passed = 0usize; - let mut skipped = 0usize; - let mut failures: Vec = Vec::new(); - - for entry in &entries { - let path = entry.path(); - let name = path.file_name().unwrap().to_string_lossy().to_string(); - - if KNOWN_DRIFT_FIXTURES.iter().any(|&s| s == name) { - eprintln!("[SKIP] {} (known drift)", name); - skipped += 1; - continue; - } - - let json = std::fs::read_to_string(&path).expect("read fixture file"); - let fixture: RuleFixture = serde_json::from_str(&json) - .unwrap_or_else(|e| panic!("JSON parse error in {}: {}", name, e)); - - let opts = fixture.options.clone().unwrap_or_default(); - let result = reduce_execution_with_rules(fixture.input.clone(), &rules, &opts); - - if result.inline_text.trim() == fixture.expected_output.trim() { - passed += 1; - } else { - let msg = format!( - "[FAIL] {}\n desc: {}\n expected: {:?}\n actual: {:?}", - name, - fixture.description.as_deref().unwrap_or("(none)"), - fixture.expected_output.trim(), - result.inline_text.trim() - ); - eprintln!("{}", msg); - failures.push(name); - } - } - - eprintln!( - "\ntokenjuice integration: {} passed, {} skipped, {} failed", - passed, - skipped, - failures.len() - ); - - assert!( - failures.is_empty(), - "{} fixture(s) failed: {}", - failures.len(), - failures.join(", ") - ); -} diff --git a/vendor/tinyjuice b/vendor/tinyjuice deleted file mode 160000 index e6848ed87d..0000000000 --- a/vendor/tinyjuice +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e6848ed87d5d661073e9eab87b6b385c3373ee38