From 41c355057c1cab5ebc52448f4a2626968a61def1 Mon Sep 17 00:00:00 2001 From: Kalev Lillemets Date: Fri, 3 Apr 2026 20:44:57 +0300 Subject: [PATCH] wip: capture OpenViking repo state --- README.md | 2 + docs/en/guides/06-mcp-integration.md | 5 + docs/en/guides/09-codex-setup.md | 204 ++++++++++++ .../.claude-plugin/marketplace.json | 17 + examples/claude-memory-plugin/README.md | 35 +- examples/claude-memory-plugin/hooks/common.sh | 274 +++++++++++++++- .../claude-memory-plugin/hooks/hooks.json | 2 +- .../claude-memory-plugin/hooks/session-end.sh | 16 +- .../hooks/session-start.sh | 49 ++- examples/claude-memory-plugin/hooks/stop.sh | 9 +- .../hooks/user-prompt-submit.sh | 2 +- .../claude-memory-plugin/scripts/ov_memory.py | 190 ++++++++++- examples/common/recipe.py | 192 ++++++++++- examples/mcp-query/README.md | 103 +++++- examples/mcp-query/server.py | 304 ++++++++++++++---- openviking/retrieve/hierarchical_retriever.py | 2 + openviking_cli/retrieve/types.py | 6 + 17 files changed, 1287 insertions(+), 125 deletions(-) create mode 100644 docs/en/guides/09-codex-setup.md create mode 100644 examples/claude-memory-plugin/.claude-plugin/marketplace.json diff --git a/README.md b/README.md index 1d6de1ab0..b2fcbc3c6 100644 --- a/README.md +++ b/README.md @@ -532,6 +532,8 @@ After integrating OpenViking: 👉 **[View: OpenCode Memory Plugin Example](examples/opencode-memory-plugin/README.md)** +👉 **[View: Codex Setup Guide](./docs/en/guides/09-codex-setup.md)** + -- ## Core Concepts diff --git a/docs/en/guides/06-mcp-integration.md b/docs/en/guides/06-mcp-integration.md index e85161017..6f8fee576 100644 --- a/docs/en/guides/06-mcp-integration.md +++ b/docs/en/guides/06-mcp-integration.md @@ -174,6 +174,10 @@ In your OpenClaw configuration (`openclaw.json` or `openclaw.yaml`): } ``` +### Codex + +If your OpenViking deployment already runs as a normal HTTP API server and you want to use it from Codex, use the local MCP bridge example described in [Codex Setup Guide](09-codex-setup.md). + ## Available MCP Tools Once connected, OpenViking exposes the following MCP tools: @@ -227,4 +231,5 @@ curl http://localhost:1933/health - [MCP Specification](https://modelcontextprotocol.io/) - [OpenViking Configuration](01-configuration.md) - [OpenViking Deployment](03-deployment.md) +- [Codex Setup Guide](09-codex-setup.md) - [Related issue: stdio contention (#473)](https://github.com/volcengine/OpenViking/issues/473) diff --git a/docs/en/guides/09-codex-setup.md b/docs/en/guides/09-codex-setup.md new file mode 100644 index 000000000..3bfd584e0 --- /dev/null +++ b/docs/en/guides/09-codex-setup.md @@ -0,0 +1,204 @@ +# Codex Setup Guide + +This guide shows how to use OpenViking with Codex when your main OpenViking server is already running over HTTP. + +## Overview + +Codex consumes external tools through MCP. + +If your OpenViking deployment is only running the normal HTTP API server (`openviking-server` on port `1933` by default), the easiest Codex setup is: + +1. Keep your existing OpenViking server running +2. Start the local MCP bridge from `examples/mcp-query/server.py` +3. Register that local bridge with Codex + +This keeps your current Claude/OpenViking setup intact and adds Codex access on top. + +## Architecture + +```text +Codex CLI + ↓ MCP +Local MCP bridge (examples/mcp-query/server.py) + ↓ HTTP +Remote OpenViking server (openviking-server) +``` + +## Prerequisites + +- OpenViking server already running and reachable over HTTP +- Python available on the Codex machine +- Codex CLI installed +- The `mcp` Python package installed + +Install `mcp` if needed: + +```bash +python -m pip install mcp +``` + +## Bridge Setup + +### 1. Set environment variables + +Use your OpenViking server URL and API key. + +```powershell +[Environment]::SetEnvironmentVariable('OV_SERVER_URL', 'http://YOUR_SERVER:1933', 'User') +[Environment]::SetEnvironmentVariable('OV_API_KEY', 'YOUR_OPENVIKING_KEY', 'User') +``` + +If your key is a normal user key, that is enough. + +If your key is a root key, also set the target tenant: + +```powershell +[Environment]::SetEnvironmentVariable('OV_ACCOUNT', 'default', 'User') +[Environment]::SetEnvironmentVariable('OV_USER', 'your-user-id', 'User') +``` + +Open a new terminal, or refresh the current session: + +```powershell +$env:OV_SERVER_URL = [Environment]::GetEnvironmentVariable('OV_SERVER_URL', 'User') +$env:OV_API_KEY = [Environment]::GetEnvironmentVariable('OV_API_KEY', 'User') +$env:OV_ACCOUNT = [Environment]::GetEnvironmentVariable('OV_ACCOUNT', 'User') +$env:OV_USER = [Environment]::GetEnvironmentVariable('OV_USER', 'User') +``` + +### 2. Start the local MCP bridge + +From the OpenViking repository: + +```powershell +cd C:\Dev\OpenViking +python examples\mcp-query\server.py --url $env:OV_SERVER_URL --api-key $env:OV_API_KEY --account $env:OV_ACCOUNT --user $env:OV_USER +``` + +If you use a normal user key instead of a root key, you can omit `--account` and `--user`. + +Expected log output includes: + +- `mode: http-bridge` +- `ov url: http://...:1933` +- `endpoint: http://127.0.0.1:2033/mcp` + +Keep this bridge terminal running while Codex uses OpenViking. + +## Codex Registration + +In another terminal: + +```powershell +codex mcp add openviking --url http://127.0.0.1:2033/mcp +codex mcp list +``` + +`codex mcp list` should show an `openviking` entry with URL `http://127.0.0.1:2033/mcp`. + +If Codex shows `Auth Unsupported` for this bridge, that is expected. Codex is talking to the local bridge without separate bridge-level authentication. + +## Available Tools + +The MCP bridge exposes: + +- `search`: semantic search in OpenViking +- `add_resource`: add files, directories, or URLs into OpenViking +- `query`: optional search + LLM answer generation +- `memory_start_session`: create a manual memory session +- `memory_add_turn`: append an important user/assistant turn +- `memory_get_session`: inspect a session +- `memory_commit_session`: extract and index memories from a session +- `memory_delete_session`: remove a session + +## First Test + +Ask Codex something explicit: + +```text +Use the openviking MCP tools to search for "OpenViking memory" and summarize what you find. +``` + +Or search for a specific project: + +```text +Use the openviking MCP tools to search for "KADE.Voice" and summarize the top matches. +``` + +## Manual Memory Workflow + +This bridge supports manual memory capture, not automatic conversation capture. + +Typical flow: + +1. `memory_start_session` +2. `memory_add_turn` +3. `memory_commit_session` + +Example: + +```text +Use the openviking MCP tools to: +1. start a memory session +2. add a turn saying I prefer Codex with OpenViking over Claude-only workflows +3. commit the session +4. tell me the session id +``` + +## Automatic Memory Behavior + +This Codex bridge does **not** automatically save every Codex conversation turn. + +Claude's OpenViking memory plugin uses dedicated lifecycle hooks such as `SessionStart`, `Stop`, and `SessionEnd`. The Codex MCP bridge does not receive those hook events, so automatic session capture is not available here yet. + +What is supported today: + +- manual memory save through MCP tools +- normal OpenViking retrieval and resource ingestion + +## Optional `query` Tool + +The `query` tool requires local LLM config in `ov.conf` because the bridge itself must call a model after search. + +If you only need search, add-resource, and manual memory tools, you do **not** need local `ov.conf`. + +If you want `query`, create a local `ov.conf` containing at least: + +```json +{ + "vlm": { + "provider": "openai", + "model": "gpt-4o", + "api_key": "your-api-key", + "api_base": "https://api.openai.com/v1" + } +} +``` + +Then start the bridge with: + +```powershell +python examples\mcp-query\server.py --url $env:OV_SERVER_URL --api-key $env:OV_API_KEY --account $env:OV_ACCOUNT --user $env:OV_USER --config .\ov.conf +``` + +## Troubleshooting + +### Codex can see the MCP server, but searches fail + +If your OpenViking key is a root key, restart the bridge with `--account` and `--user`. + +### `python examples\mcp-query\server.py --help` fails with missing `mcp` + +Install the runtime: + +```bash +python -m pip install mcp +``` + +### Codex added the MCP server, but nothing happens + +Make sure the bridge process is still running locally on `127.0.0.1:2033`. + +### I want true auto-save memory like Claude + +That needs a separate Codex-side lifecycle integration or wrapper that records turns and calls the session APIs automatically. The current guide covers the supported manual workflow only. diff --git a/examples/claude-memory-plugin/.claude-plugin/marketplace.json b/examples/claude-memory-plugin/.claude-plugin/marketplace.json new file mode 100644 index 000000000..788c04e77 --- /dev/null +++ b/examples/claude-memory-plugin/.claude-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "name": "openviking-memory", + "description": "Persistent cross-session memory for Claude Code powered by OpenViking", + "owner": { + "name": "volcengine", + "email": "openviking@volcengine.com" + }, + "plugins": [ + { + "name": "openviking-memory", + "description": "Cross-session memory plugin using OpenViking sessions, hooks, and semantic recall.", + "version": "0.1.0", + "source": "./", + "tags": ["memory", "sessions", "recall", "openviking"] + } + ] +} diff --git a/examples/claude-memory-plugin/README.md b/examples/claude-memory-plugin/README.md index 3961a97b0..da7ab210d 100644 --- a/examples/claude-memory-plugin/README.md +++ b/examples/claude-memory-plugin/README.md @@ -3,7 +3,7 @@ Claude Code memory plugin built on **OpenViking Session memory**. - Session data is accumulated during a Claude session (`Stop` hook). -- At `SessionEnd`, plugin calls `session.commit()` to trigger OpenViking memory extraction. +- At `SessionEnd`, plugin now queues a detached commit worker instead of blocking Claude exit on `session.commit()`. - Memory recall is handled by `memory-recall` skill. ## Design choices in this version @@ -14,6 +14,9 @@ Claude Code memory plugin built on **OpenViking Session memory**. - Config: **strict** - Must have `./ov.conf` in project root - Plugin state dir: `./.openviking/memory/` + - Session state: `./.openviking/memory/session_state.json` + - Detached commit queue: `./.openviking/memory/pending/` + - Detached commit logs: `./.openviking/memory/logs/` ## Structure @@ -50,7 +53,24 @@ examples/claude-memory-plugin/ - Append user + assistant summary to OpenViking session - Deduplicate by last user turn UUID - `SessionEnd` - - Commit OpenViking session to extract long-term memories + - Queue a detached commit worker so `/exit` is not blocked on remote extraction + - Persist `pending_commit_file`, `pending_commit_log`, `commit_requested_at`, and `commit_in_progress` + - Finish the real `session.commit()` in the background and write the final result into the pending state/log + +## SessionEnd behavior + +The plugin now treats `SessionEnd` as a queueing step, not a long blocking RPC. + +- The hook returns quickly with a status like `session commit queued` +- A detached Python worker continues the real OpenViking commit after Claude exits +- The live session state flips to `active=false` immediately so exit is not held open +- The background worker records final outcome in the pending state file and log + +If you want to inspect a commit after exit, check: + +- `./.openviking/memory/session_state.json` +- `./.openviking/memory/pending/.json` +- `./.openviking/memory/logs/session-end--.log` ## Skill behavior @@ -90,10 +110,19 @@ What the script does: - Generates a temporary project `./ov.conf` from source config and injects HTTP server fields. - Starts OpenViking HTTP server, runs a real `claude -p` session with this plugin, then triggers deterministic Stop + SessionEnd validation. - Verifies `session_state.json`, `ingested_turns >= 1`, and session archive file creation. +- Also tolerates the detached SessionEnd path by waiting briefly for the queued commit to finish. - Restores original `./ov.conf` when done. +## Resolved issues + +- **Health check timeout:** `_health_check()` uses 1.5s timeout (down from 3.5s). Offline detection completes in ~2.7s total. +- **SessionStart hook:** Runs synchronously with 5s timeout. Subagent sessions are filtered (exit immediately on `agent_id`). Recall is skipped when server is offline. +- **Offline buffering:** When the OV server is unreachable, turns are buffered locally and replayed on reconnect. + ## Notes - This MVP does not modify OpenViking core. - If `./ov.conf` is missing, hooks degrade safely and report status in systemMessage. -- State file: `./.openviking/memory/session_state.json` +- Primary state file: `./.openviking/memory/session_state.json` +- Detached SessionEnd state: `./.openviking/memory/pending/` +- Detached SessionEnd logs: `./.openviking/memory/logs/` diff --git a/examples/claude-memory-plugin/hooks/common.sh b/examples/claude-memory-plugin/hooks/common.sh index 661d928aa..791affd71 100755 --- a/examples/claude-memory-plugin/hooks/common.sh +++ b/examples/claude-memory-plugin/hooks/common.sh @@ -1,9 +1,14 @@ #!/usr/bin/env bash # Shared helpers for OpenViking Claude Code hooks. -set -euo pipefail +set -uo pipefail -INPUT="$(cat || true)" +# Read stdin if available (hooks receive JSON input), with timeout to avoid hang +if [[ -t 0 ]]; then + INPUT="" +else + INPUT="$(timeout 3 cat 2>/dev/null || true)" +fi for p in "$HOME/.local/bin" "$HOME/.cargo/bin" "$HOME/bin" "/usr/local/bin"; do [[ -d "$p" ]] && [[ ":$PATH:" != *":$p:"* ]] && export PATH="$p:$PATH" @@ -15,8 +20,32 @@ PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(pwd)}" STATE_DIR="$PROJECT_DIR/.openviking/memory" STATE_FILE="$STATE_DIR/session_state.json" -OV_CONF="$PROJECT_DIR/ov.conf" +LOG_DIR="$STATE_DIR/logs" +PENDING_DIR="$STATE_DIR/pending" +ARCHIVE_DIR="$STATE_DIR/archive" BRIDGE="$PLUGIN_ROOT/scripts/ov_memory.py" +PENDING_WARN_COUNT="${OPENVIKING_PENDING_WARN_COUNT:-20}" +PENDING_MAX_COUNT="${OPENVIKING_PENDING_MAX_COUNT:-60}" +PENDING_STALE_MINUTES="${OPENVIKING_PENDING_STALE_MINUTES:-360}" + +# Search for ov.conf: project dir → upward → plugin root → ~/.openviking/ +_find_ov_conf() { + local dir="$PROJECT_DIR" + local depth=0 + while [[ -n "$dir" && "$dir" != "/" && "$dir" != "." && $depth -lt 10 ]]; do + if [[ -f "$dir/ov.conf" ]]; then echo "$dir/ov.conf"; return 0; fi + local parent + parent="$(dirname "$dir")" + [[ "$parent" == "$dir" ]] && break + dir="$parent" + depth=$((depth + 1)) + done + if [[ -f "$PLUGIN_ROOT/ov.conf" ]]; then echo "$PLUGIN_ROOT/ov.conf"; return 0; fi + if [[ -f "$HOME/.openviking/ov.conf" ]]; then echo "$HOME/.openviking/ov.conf"; return 0; fi + echo "" + return 0 +} +OV_CONF="$(_find_ov_conf)" || true if command -v python3 >/dev/null 2>&1; then PYTHON_BIN="python3" @@ -75,7 +104,100 @@ _json_encode_str() { } ensure_state_dir() { - mkdir -p "$STATE_DIR" + mkdir -p "$STATE_DIR" "$LOG_DIR" "$PENDING_DIR" "$ARCHIVE_DIR" +} + +maintain_pending_queue() { + ensure_state_dir + + if [[ -z "$PYTHON_BIN" ]]; then + echo '{"ok": false, "error": "python not found"}' + return 1 + fi + + "$PYTHON_BIN" - "$STATE_FILE" "$PENDING_DIR" "$ARCHIVE_DIR" "$PENDING_WARN_COUNT" "$PENDING_MAX_COUNT" "$PENDING_STALE_MINUTES" <<'PY' +import json +import shutil +import sys +import time +from pathlib import Path + + +state_file, pending_dir, archive_dir, warn_count, max_count, stale_minutes = sys.argv[1:7] +pending_path = Path(pending_dir) +archive_root = Path(archive_dir) +state_path = Path(state_file) +now = time.time() + +warn_count = max(int(warn_count or 20), 1) +max_count = max(int(max_count or 60), warn_count) +stale_seconds = max(int(stale_minutes or 360), 1) * 60 + +current_pending = "" +if state_path.exists(): + try: + state = json.loads(state_path.read_text(encoding="utf-8")) + current_pending = str(state.get("pending_commit_file") or "").strip() + except Exception: + current_pending = "" + +pending_files = sorted( + [p for p in pending_path.glob("*.json") if p.is_file()], + key=lambda p: p.stat().st_mtime, +) +count_before = len(pending_files) +archived = [] +warn_reasons = [] + +eligible = [] +for item in pending_files: + age_seconds = max(0, int(now - item.stat().st_mtime)) + if current_pending and str(item) == current_pending: + continue + if age_seconds >= stale_seconds: + eligible.append((item, age_seconds)) + +if count_before > max_count and not eligible: + warn_reasons.append("backlog-over-limit-without-stale-files") + +if eligible: + stamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime(now)) + archive_path = archive_root / f"pending-{stamp}" + archive_path.mkdir(parents=True, exist_ok=True) + for item, age_seconds in eligible: + target = archive_path / item.name + shutil.move(str(item), str(target)) + archived.append( + { + "name": item.name, + "path": str(target), + "age_seconds": age_seconds, + } + ) + +pending_after = sorted( + [p for p in pending_path.glob("*.json") if p.is_file()], + key=lambda p: p.stat().st_mtime, +) +count_after = len(pending_after) +if count_after >= warn_count: + warn_reasons.append("backlog-warning-threshold") + +print( + json.dumps( + { + "ok": True, + "pending_count_before": count_before, + "pending_count_after": count_after, + "archived_count": len(archived), + "archive_dir": str(archive_path) if archived else "", + "warn": bool(warn_reasons), + "warn_reasons": warn_reasons, + "archived": archived[:20], + } + ) +) +PY } run_bridge() { @@ -89,8 +211,148 @@ run_bridge() { fi ensure_state_dir - "$PYTHON_BIN" "$BRIDGE" \ + # OpenViking logs to stdout; extract only the last JSON line + local raw + raw="$("$PYTHON_BIN" "$BRIDGE" \ --project-dir "$PROJECT_DIR" \ --state-file "$STATE_FILE" \ - "$@" + --ov-conf "$OV_CONF" \ + "$@" 2>/dev/null)" + # Return only the last line that starts with '{' + printf '%s\n' "$raw" | grep '^{' | tail -1 +} + +queue_session_end_commit() { + if [[ -z "$PYTHON_BIN" ]]; then + echo '{"ok": false, "error": "python not found"}' + return 1 + fi + if [[ ! -f "$BRIDGE" ]]; then + echo '{"ok": false, "error": "bridge script not found"}' + return 1 + fi + if [[ ! -f "$STATE_FILE" ]]; then + echo '{"ok": false, "error": "state file not found"}' + return 1 + fi + + ensure_state_dir + + "$PYTHON_BIN" - "$PYTHON_BIN" "$BRIDGE" "$PROJECT_DIR" "$STATE_FILE" "$OV_CONF" "$LOG_DIR" "$PENDING_DIR" <<'PY' +import json +import os +import subprocess +import sys +import time +from pathlib import Path + + +python_bin, bridge, project_dir, state_file, ov_conf, log_dir, pending_dir = sys.argv[1:8] +state_path = Path(state_file) + +try: + state = json.loads(state_path.read_text(encoding="utf-8")) +except Exception as exc: # noqa: BLE001 + print(json.dumps({"ok": False, "error": f"failed to read state file: {exc}"})) + raise SystemExit(1) + +session_id = str(state.get("session_id") or "").strip() +if not state.get("active") or not session_id: + print( + json.dumps( + { + "ok": True, + "queued": False, + "status_line": "[openviking-memory] no active session", + } + ) + ) + raise SystemExit(0) + +now = int(time.time()) +log_dir_path = Path(log_dir) +pending_dir_path = Path(pending_dir) +log_dir_path.mkdir(parents=True, exist_ok=True) +pending_dir_path.mkdir(parents=True, exist_ok=True) + +pending_path = pending_dir_path / f"{session_id}.json" +log_path = log_dir_path / f"session-end-{session_id}-{now}.log" + +pending_state = dict(state) +pending_state["commit_requested_at"] = now +pending_state["commit_mode"] = "detached" +pending_state["commit_in_progress"] = False +pending_state["pending_commit_log"] = str(log_path) +pending_state["last_commit_error"] = "" +pending_path.write_text( + json.dumps(pending_state, ensure_ascii=False, indent=2), + encoding="utf-8", +) + +command = [ + python_bin, + bridge, + "--project-dir", + project_dir, + "--state-file", + str(pending_path), +] +if ov_conf: + command += ["--ov-conf", ov_conf] +command += ["session-end"] + +header = ( + f"ts={time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(now))}\n" + f"session_id={session_id}\n" + f"mode=detached\n" + f"pending_file={pending_path}\n" + f"command={' '.join(command)}\n" + "---\n" +) + +try: + with open(log_path, "ab", buffering=0) as log_handle: + log_handle.write(header.encode("utf-8")) + kwargs = { + "stdin": subprocess.DEVNULL, + "stdout": log_handle, + "stderr": subprocess.STDOUT, + "cwd": project_dir, + "close_fds": True, + } + if os.name == "nt": + kwargs["creationflags"] = 0x00000008 | 0x00000200 | 0x08000000 + else: + kwargs["start_new_session"] = True + subprocess.Popen(command, **kwargs) +except Exception as exc: # noqa: BLE001 + print(json.dumps({"ok": False, "error": f"failed to queue session-end commit: {exc}"})) + raise SystemExit(1) + +live_state = dict(state) +live_state["active"] = False +live_state["commit_requested_at"] = now +live_state["commit_mode"] = "detached" +live_state["commit_in_progress"] = False +live_state["pending_commit_file"] = str(pending_path) +live_state["pending_commit_log"] = str(log_path) +live_state["last_commit_error"] = "" +state_path.write_text( + json.dumps(live_state, ensure_ascii=False, indent=2), + encoding="utf-8", +) + +print( + json.dumps( + { + "ok": True, + "queued": True, + "session_id": session_id, + "pending_file": str(pending_path), + "log_file": str(log_path), + "status_line": f"[openviking-memory] session commit queued id={session_id}", + } + ) +) +PY } diff --git a/examples/claude-memory-plugin/hooks/hooks.json b/examples/claude-memory-plugin/hooks/hooks.json index bfaecf328..5f5fc62ef 100644 --- a/examples/claude-memory-plugin/hooks/hooks.json +++ b/examples/claude-memory-plugin/hooks/hooks.json @@ -7,7 +7,7 @@ { "type": "command", "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/session-start.sh", - "timeout": 12 + "timeout": 5 } ] } diff --git a/examples/claude-memory-plugin/hooks/session-end.sh b/examples/claude-memory-plugin/hooks/session-end.sh index 3640517f3..c1d8281d6 100755 --- a/examples/claude-memory-plugin/hooks/session-end.sh +++ b/examples/claude-memory-plugin/hooks/session-end.sh @@ -4,11 +4,23 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/common.sh" -if [[ ! -f "$OV_CONF" || ! -f "$STATE_FILE" ]]; then +# Skip subagent sessions +AGENT_ID="$(_json_val "$INPUT" "agent_id" "")" +if [[ -n "$AGENT_ID" ]]; then exit 0 fi -OUT="$(run_bridge session-end 2>/dev/null || true)" +if [[ -z "$OV_CONF" || ! -f "$OV_CONF" || ! -f "$STATE_FILE" ]]; then + exit 0 +fi + +# Offline sessions: close directly (fast, no network), skip detached queue +SESSION_MODE="$(_json_val "$(cat "$STATE_FILE" 2>/dev/null || true)" "mode" "")" +if [[ "$SESSION_MODE" == "offline" ]]; then + OUT="$(run_bridge session-end 2>/dev/null || true)" +else + OUT="$(queue_session_end_commit 2>/dev/null || run_bridge session-end 2>/dev/null || true)" +fi STATUS="$(_json_val "$OUT" "status_line" "")" if [[ -n "$STATUS" ]]; then diff --git a/examples/claude-memory-plugin/hooks/session-start.sh b/examples/claude-memory-plugin/hooks/session-start.sh index 5811e8b2f..ae2f7595d 100755 --- a/examples/claude-memory-plugin/hooks/session-start.sh +++ b/examples/claude-memory-plugin/hooks/session-start.sh @@ -4,24 +4,61 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/common.sh" -if [[ ! -f "$OV_CONF" ]]; then - msg='[openviking-memory] ERROR: ./ov.conf not found (strict mode)' +# Skip subagent sessions — they create empty server overhead +AGENT_ID="$(_json_val "$INPUT" "agent_id" "")" +if [[ -n "$AGENT_ID" ]]; then + echo '{}' + exit 0 +fi + +if [[ -z "$OV_CONF" || ! -f "$OV_CONF" ]]; then + msg='[openviking-memory] ERROR: ov.conf not found (checked project dir, parents, plugin root, ~/.openviking/)' json_msg=$(_json_encode_str "$msg") echo "{\"systemMessage\": $json_msg}" exit 0 fi +PENDING_STATUS="$(maintain_pending_queue 2>/dev/null || true)" OUT="$(run_bridge session-start 2>/dev/null || true)" OK="$(_json_val "$OUT" "ok" "false")" STATUS="$(_json_val "$OUT" "status_line" "[openviking-memory] initialization failed")" ADDL="$(_json_val "$OUT" "additional_context" "")" +ARCHIVED_COUNT="$(_json_val "$PENDING_STATUS" "archived_count" "0")" +PENDING_AFTER="$(_json_val "$PENDING_STATUS" "pending_count_after" "0")" +WARN_BACKLOG="$(_json_val "$PENDING_STATUS" "warn" "false")" + +if [[ "$ARCHIVED_COUNT" != "0" ]]; then + STATUS="$STATUS archived_stale_pending=$ARCHIVED_COUNT" +fi +if [[ "$WARN_BACKLOG" == "true" && "$PENDING_AFTER" != "0" ]]; then + STATUS="$STATUS pending_backlog=$PENDING_AFTER" +fi json_status=$(_json_encode_str "$STATUS") -if [[ "$OK" == "true" && -n "$ADDL" ]]; then - json_addl=$(_json_encode_str "$ADDL") - echo "{\"systemMessage\": $json_status, \"hookSpecificOutput\": {\"hookEventName\": \"SessionStart\", \"additionalContext\": $json_addl}}" - exit 0 +MODE="$(_json_val "$OUT" "mode" "")" + +if [[ "$OK" == "true" ]]; then + # Recall recent session history (skip when offline — server unreachable) + HISTORY="" + if [[ "$MODE" != "offline" ]]; then + HISTORY="$(run_bridge recall --query "recent decisions changes fixes patterns" --top-k 5 2>/dev/null || true)" + fi + HIST_TEXT="" + if [[ -n "$HISTORY" ]]; then + HIST_TEXT="$(_json_val "$HISTORY" "formatted" "")" + fi + + FULL_CTX="${ADDL}" + if [[ -n "$HIST_TEXT" ]]; then + FULL_CTX="${FULL_CTX}\n\nOpenViking session history:\n${HIST_TEXT}" + fi + + if [[ -n "$FULL_CTX" ]]; then + json_addl=$(_json_encode_str "$FULL_CTX") + echo "{\"systemMessage\": $json_status, \"hookSpecificOutput\": {\"hookEventName\": \"SessionStart\", \"additionalContext\": $json_addl}}" + exit 0 + fi fi echo "{\"systemMessage\": $json_status}" diff --git a/examples/claude-memory-plugin/hooks/stop.sh b/examples/claude-memory-plugin/hooks/stop.sh index 67fb15e22..963c65b9b 100755 --- a/examples/claude-memory-plugin/hooks/stop.sh +++ b/examples/claude-memory-plugin/hooks/stop.sh @@ -4,13 +4,20 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/common.sh" +# Skip subagent sessions +AGENT_ID="$(_json_val "$INPUT" "agent_id" "")" +if [[ -n "$AGENT_ID" ]]; then + echo '{}' + exit 0 +fi + STOP_HOOK_ACTIVE="$(_json_val "$INPUT" "stop_hook_active" "false")" if [[ "$STOP_HOOK_ACTIVE" == "true" ]]; then echo '{}' exit 0 fi -if [[ ! -f "$OV_CONF" || ! -f "$STATE_FILE" ]]; then +if [[ -z "$OV_CONF" || ! -f "$OV_CONF" || ! -f "$STATE_FILE" ]]; then echo '{}' exit 0 fi diff --git a/examples/claude-memory-plugin/hooks/user-prompt-submit.sh b/examples/claude-memory-plugin/hooks/user-prompt-submit.sh index f5a080914..42efc849d 100755 --- a/examples/claude-memory-plugin/hooks/user-prompt-submit.sh +++ b/examples/claude-memory-plugin/hooks/user-prompt-submit.sh @@ -10,7 +10,7 @@ if [[ -z "$PROMPT" || ${#PROMPT} -lt 10 ]]; then exit 0 fi -if [[ ! -f "$OV_CONF" || ! -f "$STATE_FILE" ]]; then +if [[ -z "$OV_CONF" || ! -f "$OV_CONF" || ! -f "$STATE_FILE" ]]; then echo '{}' exit 0 fi diff --git a/examples/claude-memory-plugin/scripts/ov_memory.py b/examples/claude-memory-plugin/scripts/ov_memory.py index a249c8e87..74782c5a1 100755 --- a/examples/claude-memory-plugin/scripts/ov_memory.py +++ b/examples/claude-memory-plugin/scripts/ov_memory.py @@ -16,6 +16,7 @@ import shutil import subprocess import time +import uuid from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional @@ -30,9 +31,19 @@ class BackendInfo: local_data_path: str = "" +def _expand_env_vars(obj: Any) -> Any: + if isinstance(obj, str): + return os.path.expandvars(obj) + if isinstance(obj, dict): + return {k: _expand_env_vars(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_expand_env_vars(v) for v in obj] + return obj + + def _load_json(path: Path) -> Dict[str, Any]: with open(path, "r", encoding="utf-8") as f: - return json.load(f) + return _expand_env_vars(json.load(f)) def _save_json(path: Path, data: Dict[str, Any]) -> None: @@ -50,7 +61,7 @@ def _load_state(path: Path) -> Dict[str, Any]: return {} -def _health_check(url: str, timeout: float = 1.2) -> bool: +def _health_check(url: str, timeout: float = 1.5) -> bool: try: with request.urlopen(f"{url.rstrip('/')}/health", timeout=timeout) as resp: if resp.status != 200: @@ -88,6 +99,8 @@ def detect_backend(project_dir: Path, ov_conf: Dict[str, Any]) -> BackendInfo: if _health_check(url): return BackendInfo(mode="http", url=url, api_key=str(api_key)) + # Server configured but unreachable — don't fall back to local + return BackendInfo(mode="offline", url=url, api_key=str(api_key)) return BackendInfo( mode="local", @@ -108,6 +121,8 @@ def __enter__(self) -> "OVClient": self.client = SyncHTTPClient( url=self.backend.url, api_key=self.backend.api_key or None, + account="default", + user="kalev", ) self.client.initialize() return self @@ -429,9 +444,61 @@ def _build_backend_from_state_or_detect( return detect_backend(project_dir, ov_conf) +def _resolve_ov_conf(args: argparse.Namespace) -> Path: + if getattr(args, "ov_conf", "") and args.ov_conf: + return Path(args.ov_conf).resolve() + return Path(args.project_dir).resolve() / "ov.conf" + + +def _offline_turns_path(state_dir: Path, session_id: str) -> Path: + return state_dir / "offline" / f"{session_id}.turns.jsonl" + + +def _replay_offline_sessions( + backend: BackendInfo, ov_conf_path: Path, state_dir: Path, +) -> int: + """Replay buffered offline sessions to the server. Returns count replayed.""" + offline_dir = state_dir / "offline" + if not offline_dir.is_dir(): + return 0 + replayed = 0 + for turns_file in sorted(offline_dir.glob("*.turns.jsonl")): + turns: List[Dict[str, Any]] = [] + try: + for line in turns_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line: + turns.append(json.loads(line)) + except Exception: + continue + if not turns: + turns_file.unlink(missing_ok=True) + continue + try: + with OVClient(backend, ov_conf_path) as cli: + sess = cli.create_session() + sid = _as_text(sess.get("session_id")) + if not sid: + continue + for t in turns: + cli.add_message(sid, t["role"], t["content"]) + cli.commit_session(sid) + turns_file.unlink(missing_ok=True) + replayed += 1 + except Exception: + break # server may have gone offline mid-replay + # Clean up empty offline dir + try: + if offline_dir.is_dir() and not list(offline_dir.iterdir()): + offline_dir.rmdir() + except Exception: + pass + return replayed + + def cmd_session_start(args: argparse.Namespace) -> Dict[str, Any]: project_dir = Path(args.project_dir).resolve() - ov_conf_path = project_dir / "ov.conf" + ov_conf_path = _resolve_ov_conf(args) state_file = Path(args.state_file) if not ov_conf_path.exists(): @@ -441,9 +508,45 @@ def cmd_session_start(args: argparse.Namespace) -> Dict[str, Any]: "error": "ov.conf not found", } + state_dir = state_file.parent ov_conf = _load_json(ov_conf_path) backend = detect_backend(project_dir, ov_conf) + if backend.mode == "offline": + # Create a local session so turns are still captured + session_id = str(uuid.uuid4()) + state = { + "active": True, + "project_dir": str(project_dir), + "ov_conf": str(ov_conf_path), + "mode": "offline", + "url": backend.url, + "api_key": backend.api_key, + "local_data_path": "", + "session_id": session_id, + "last_turn_uuid": "", + "ingested_turns": 0, + "started_at": int(time.time()), + } + _save_json(state_file, state) + # Ensure offline turns dir exists + _offline_turns_path(state_dir, session_id).parent.mkdir( + parents=True, exist_ok=True, + ) + return { + "ok": True, + "mode": "offline", + "session_id": session_id, + "status_line": f"[openviking-memory] server offline ({backend.url}), buffering locally", + "additional_context": ( + "OpenViking server is offline. Turns will be buffered locally " + "and uploaded when the server is reachable again." + ), + } + + # Server is online — replay any buffered offline sessions first + replayed = _replay_offline_sessions(backend, ov_conf_path, state_dir) + with OVClient(backend, ov_conf_path) as cli: session = cli.create_session() session_id = _as_text(session.get("session_id")) @@ -468,6 +571,8 @@ def cmd_session_start(args: argparse.Namespace) -> Dict[str, Any]: status = f"[openviking-memory] mode={backend.mode} session={session_id}" if backend.mode == "http": status += f" server={backend.url}" + if replayed > 0: + status += f" replayed_offline={replayed}" additional = ( "OpenViking memory is active. " @@ -485,7 +590,7 @@ def cmd_session_start(args: argparse.Namespace) -> Dict[str, Any]: def cmd_ingest_stop(args: argparse.Namespace) -> Dict[str, Any]: project_dir = Path(args.project_dir).resolve() - ov_conf_path = project_dir / "ov.conf" + ov_conf_path = _resolve_ov_conf(args) state_file = Path(args.state_file) transcript = Path(args.transcript_path) @@ -506,9 +611,6 @@ def cmd_ingest_stop(args: argparse.Namespace) -> Dict[str, Any]: if _as_text(turn.get("turn_uuid")) == _as_text(state.get("last_turn_uuid")): return {"ok": True, "ingested": False, "reason": "duplicate turn"} - ov_conf = _load_json(ov_conf_path) - backend = _build_backend_from_state_or_detect(state, project_dir, ov_conf) - summary = summarize_turn(turn) user_text = _as_text(turn.get("user_text")) @@ -520,8 +622,34 @@ def cmd_ingest_stop(args: argparse.Namespace) -> Dict[str, Any]: if assistant_excerpt: assistant_msg += f"\n\nAssistant excerpt:\n{_short(assistant_excerpt, 1500)}" + session_id = _as_text(state.get("session_id")) + state_dir = state_file.parent + + # Offline mode: buffer turns locally + if _as_text(state.get("mode")) == "offline": + turns_path = _offline_turns_path(state_dir, session_id) + turns_path.parent.mkdir(parents=True, exist_ok=True) + with open(turns_path, "a", encoding="utf-8") as f: + f.write(json.dumps({"role": "user", "content": user_text}) + "\n") + f.write(json.dumps({"role": "assistant", "content": assistant_msg}) + "\n") + state["last_turn_uuid"] = _as_text(turn.get("turn_uuid")) + state["ingested_turns"] = int(state.get("ingested_turns", 0)) + 1 + state["last_ingested_at"] = int(time.time()) + _save_json(state_file, state) + return { + "ok": True, + "ingested": True, + "mode": "offline", + "session_id": session_id, + "turn_uuid": turn.get("turn_uuid"), + "ingested_turns": state.get("ingested_turns"), + } + + # Online mode: send to server + ov_conf = _load_json(ov_conf_path) + backend = _build_backend_from_state_or_detect(state, project_dir, ov_conf) + with OVClient(backend, ov_conf_path) as cli: - session_id = _as_text(state.get("session_id")) cli.add_message(session_id, "user", user_text) cli.add_message(session_id, "assistant", assistant_msg) @@ -537,7 +665,7 @@ def cmd_ingest_stop(args: argparse.Namespace) -> Dict[str, Any]: return { "ok": True, "ingested": True, - "session_id": state.get("session_id"), + "session_id": session_id, "turn_uuid": turn.get("turn_uuid"), "ingested_turns": state.get("ingested_turns"), } @@ -545,7 +673,7 @@ def cmd_ingest_stop(args: argparse.Namespace) -> Dict[str, Any]: def cmd_session_end(args: argparse.Namespace) -> Dict[str, Any]: project_dir = Path(args.project_dir).resolve() - ov_conf_path = project_dir / "ov.conf" + ov_conf_path = _resolve_ov_conf(args) state_file = Path(args.state_file) state = _load_state(state_file) @@ -556,6 +684,25 @@ def cmd_session_end(args: argparse.Namespace) -> Dict[str, Any]: "status_line": "[openviking-memory] no active session", } + # Offline session: turns are already buffered, mark inactive + # They'll be replayed on next online session-start + if _as_text(state.get("mode")) == "offline": + session_id = _as_text(state.get("session_id")) + turns = int(state.get("ingested_turns", 0)) + state["active"] = False + state["committed_at"] = int(time.time()) + _save_json(state_file, state) + return { + "ok": True, + "committed": False, + "mode": "offline", + "status_line": ( + f"[openviking-memory] offline session closed" + f" id={session_id} buffered_turns={turns}" + f" (will upload when server is back)" + ), + } + if not ov_conf_path.exists(): return { "ok": False, @@ -567,12 +714,28 @@ def cmd_session_end(args: argparse.Namespace) -> Dict[str, Any]: ov_conf = _load_json(ov_conf_path) backend = _build_backend_from_state_or_detect(state, project_dir, ov_conf) - with OVClient(backend, ov_conf_path) as cli: - result = cli.commit_session(_as_text(state.get("session_id"))) + state["commit_requested_at"] = int(state.get("commit_requested_at") or time.time()) + state["commit_started_at"] = int(time.time()) + state["commit_in_progress"] = True + state["commit_attempts"] = int(state.get("commit_attempts", 0)) + 1 + state["last_commit_error"] = "" + _save_json(state_file, state) + + try: + with OVClient(backend, ov_conf_path) as cli: + result = cli.commit_session(_as_text(state.get("session_id"))) + except Exception as exc: + state["commit_in_progress"] = False + state["last_commit_error"] = str(exc) + state["last_commit_failed_at"] = int(time.time()) + _save_json(state_file, state) + raise state["active"] = False + state["commit_in_progress"] = False state["committed_at"] = int(time.time()) state["commit_result"] = result + state["last_commit_error"] = "" _save_json(state_file, state) extracted = int(result.get("memories_extracted", 0)) if isinstance(result, dict) else 0 @@ -592,7 +755,7 @@ def cmd_session_end(args: argparse.Namespace) -> Dict[str, Any]: def cmd_recall(args: argparse.Namespace) -> int: project_dir = Path(args.project_dir).resolve() - ov_conf_path = project_dir / "ov.conf" + ov_conf_path = _resolve_ov_conf(args) state_file = Path(args.state_file) query = _as_text(args.query) @@ -665,6 +828,7 @@ def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="OpenViking memory bridge") parser.add_argument("--project-dir", required=True, help="Claude project directory") parser.add_argument("--state-file", required=True, help="Plugin state file path") + parser.add_argument("--ov-conf", default="", help="Path to ov.conf (overrides project-dir lookup)") sub = parser.add_subparsers(dest="command", required=True) diff --git a/examples/common/recipe.py b/examples/common/recipe.py index 8a1f9b539..8177689a2 100644 --- a/examples/common/recipe.py +++ b/examples/common/recipe.py @@ -6,6 +6,7 @@ import json import time +from pathlib import Path from typing import Any, Dict, List, Optional import requests @@ -24,29 +25,105 @@ class Recipe: 3. Return generated answer with sources """ - def __init__(self, config_path: str = "./ov.conf", data_path: str = "./data"): + def __init__( + self, + config_path: Optional[str] = "./ov.conf", + data_path: str = "./data", + server_url: Optional[str] = None, + api_key: Optional[str] = None, + account: Optional[str] = None, + user: Optional[str] = None, + agent_id: Optional[str] = None, + timeout: float = 60.0, + llm_api_key: Optional[str] = None, + ): """ Initialize RAG pipeline Args: - config_path: Path to config file with LLM settings - data_path: Path to OpenViking data directory + config_path: Optional path to config file with LLM settings + data_path: Path to local OpenViking data directory + server_url: Optional remote OpenViking HTTP server URL + api_key: Optional OpenViking HTTP API key + account: Optional OpenViking account header + user: Optional OpenViking user header + agent_id: Optional OpenViking agent header + timeout: Timeout for both OpenViking HTTP calls and LLM calls + llm_api_key: Optional override for the query LLM API key """ - # Load configuration - with open(config_path, "r") as f: - self.config_dict = json.load(f) + self.config_path = config_path + self.data_path = data_path + self.server_url = server_url + self.timeout = timeout + self.mode = "http" if server_url else "local" + self.config_dict: Dict[str, Any] = {} + + if config_path and Path(config_path).is_file(): + with open(config_path, "r", encoding="utf-8") as f: + self.config_dict = json.load(f) + elif config_path and not server_url: + raise FileNotFoundError(f"Config file not found: {config_path}") # Extract LLM config self.vlm_config = self.config_dict.get("vlm", {}) self.api_base = self.vlm_config.get("api_base") - self.api_key = self.vlm_config.get("api_key") + self.api_key = llm_api_key or self.vlm_config.get("api_key") self.model = self.vlm_config.get("model") + self.extra_headers = dict(self.vlm_config.get("extra_headers") or {}) # Initialize OpenViking client - config = OpenVikingConfig.from_dict(self.config_dict) - self.client = ov.SyncOpenViking(path=data_path, config=config) + if server_url: + self.client = ov.SyncHTTPClient( + url=server_url, + api_key=api_key, + agent_id=agent_id, + account=account, + user=user, + timeout=timeout, + ) + else: + config = OpenVikingConfig.from_dict(self.config_dict) + self.client = ov.SyncOpenViking(path=data_path, config=config) self.client.initialize() + @property + def query_ready(self) -> bool: + """Whether the query tool has enough LLM config to answer questions.""" + return bool(self.api_base and self.model) + + def _get_result_timestamps(self, resource: Any) -> Dict[str, Optional[str]]: + """Resolve timestamps for a search result. + + Prefer metadata already present on the matched context. If the remote server + does not expose timestamps in search results yet, fall back to filesystem + metadata from `stat()` and map `modTime` to `updated_at`. + """ + created_at = getattr(resource, "created_at", None) + updated_at = getattr(resource, "updated_at", None) + + if created_at or updated_at: + return { + "created_at": created_at, + "updated_at": updated_at, + } + + try: + stat_result = self.client.stat(resource.uri) + except Exception: + return { + "created_at": None, + "updated_at": None, + } + + return { + "created_at": stat_result.get("created_at") or stat_result.get("createTime"), + "updated_at": ( + stat_result.get("updated_at") + or stat_result.get("modTime") + or stat_result.get("modified_at") + ), + } + def search( self, query: str, @@ -70,19 +147,23 @@ def search( # Search all resources or specific target # `find` has better performance, but not so smart - results = self.client.search(query, target_uri=target_uri, score_threshold=score_threshold) + search_target = target_uri or "" + results = self.client.search(query, target_uri=search_target, score_threshold=score_threshold) # Extract top results search_results = [] for _i, resource in enumerate( results.resources[:top_k] + results.memories[:top_k] ): # ignore SKILLs for mvp + timestamps = self._get_result_timestamps(resource) try: content = self.client.read(resource.uri) search_results.append( { "uri": resource.uri, "score": resource.score, + "created_at": timestamps["created_at"], + "updated_at": timestamps["updated_at"], "content": content, } ) @@ -97,6 +178,8 @@ def search( { "uri": resource.uri, "score": resource.score, + "created_at": timestamps["created_at"], + "updated_at": timestamps["updated_at"], "content": f"[Directory Abstract] {abstract}", } ) @@ -125,9 +208,17 @@ def call_llm( Returns: LLM response text """ - url = f"{self.api_base}/chat/completions" + if not self.query_ready: + raise RuntimeError( + "The query tool requires a local ov.conf with vlm.api_base and vlm.model. " + "Start the MCP bridge with --config /path/to/ov.conf, or use the search tool." + ) + + url = f"{self.api_base.rstrip('/')}/chat/completions" - headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"} + headers = {"Content-Type": "application/json", **self.extra_headers} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" payload = { "model": self.model, @@ -137,7 +228,7 @@ def call_llm( } print(f"🤖 Calling LLM: {self.model}") - response = requests.post(url, json=payload, headers=headers) + response = requests.post(url, json=payload, headers=headers, timeout=self.timeout) response.raise_for_status() result = response.json() @@ -145,6 +236,81 @@ def call_llm( return answer + def add_resource(self, resource_path: str, wait_timeout: float = 300) -> str: + """ + Add a resource through the configured OpenViking client. + + In HTTP mode the underlying client automatically uploads local files to the + remote OpenViking server before indexing. + """ + result = self.client.add_resource(path=resource_path) + + if result and "root_uri" in result: + root_uri = result["root_uri"] + self.client.wait_processed(timeout=wait_timeout) + return f"Resource added and indexed: {root_uri}" + if result and result.get("status") == "error": + errors = result.get("errors", [])[:3] + error_msg = "\n".join(f" - {e}" for e in errors) + return f"Resource had parsing issues:\n{error_msg}\nSome content may still be searchable." + return "Failed to add resource." + + def create_memory_session(self) -> Dict[str, Any]: + """Create a new OpenViking session for manual memory capture.""" + return self.client.create_session() + + def get_memory_session(self, session_id: str) -> Dict[str, Any]: + """Inspect an existing OpenViking memory session.""" + return self.client.get_session(session_id) + + def delete_memory_session(self, session_id: str) -> Dict[str, Any]: + """Delete an existing OpenViking memory session.""" + self.client.delete_session(session_id) + return {"session_id": session_id, "deleted": True} + + def add_memory_turn( + self, + session_id: str, + user_message: Optional[str] = None, + assistant_message: Optional[str] = None, + note: Optional[str] = None, + ) -> Dict[str, Any]: + """Append a user/assistant turn to an OpenViking session.""" + user_text = (user_message or "").strip() + assistant_text = (assistant_message or "").strip() + note_text = (note or "").strip() + + if not any((user_text, assistant_text, note_text)): + raise ValueError("At least one of user_message, assistant_message, or note must be set.") + + if user_text: + self.client.add_message(session_id, role="user", content=user_text) + + assistant_parts = [] + if assistant_text: + assistant_parts.append(assistant_text) + if note_text: + assistant_parts.append(f"[note]\n{note_text}") + if assistant_parts: + self.client.add_message( + session_id, + role="assistant", + content="\n\n".join(assistant_parts), + ) + + session = self.client.get_session(session_id) + return { + "session_id": session_id, + "message_count": session.get("message_count", 0), + } + + def commit_memory_session(self, session_id: str) -> Dict[str, Any]: + """Commit a session so OpenViking extracts and indexes memories.""" + result = self.client.commit_session(session_id) + if isinstance(result, dict): + result.setdefault("session_id", session_id) + return result + def query( self, user_query: str, diff --git a/examples/mcp-query/README.md b/examples/mcp-query/README.md index f66e4da6d..e25014001 100644 --- a/examples/mcp-query/README.md +++ b/examples/mcp-query/README.md @@ -1,30 +1,75 @@ -# OpenViking MCP Server +# OpenViking MCP Bridge -MCP (Model Context Protocol) HTTP server that exposes OpenViking RAG capabilities as tools. +MCP (Model Context Protocol) bridge that exposes OpenViking capabilities as MCP tools. + +It supports two deployment modes: + +- Embedded/local mode: the bridge opens a local OpenViking workspace directly +- Remote HTTP bridge mode: the bridge talks to an existing `openviking-server` over HTTP + +For Codex users with OpenViking already running on another machine, the remote HTTP bridge mode is the recommended setup. ## Tools | Tool | Description | |------|-------------| -| `query` | Full RAG pipeline — search + LLM answer generation | +| `query` | Full RAG pipeline: search + LLM answer generation. Optional; requires local `ov.conf` with `vlm` configured | | `search` | Semantic search only, returns matching documents | -| `add_resource` | Add files, directories, or URLs to the database | +| `add_resource` | Add files, directories, or URLs to OpenViking | +| `memory_start_session` | Create a manual OpenViking memory session | +| `memory_add_turn` | Append an important user/assistant turn into that session | +| `memory_get_session` | Inspect a memory session | +| `memory_commit_session` | Commit a memory session so OpenViking extracts memories | +| `memory_delete_session` | Delete a memory session | ## Quick Start +### Remote OpenViking HTTP Server + +If OpenViking is already running elsewhere, start the MCP bridge locally and point it at that server: + +```bash +# Install dependencies +uv sync + +# Start the bridge against an existing OpenViking server +uv run server.py --url http://YOUR_SERVER:1933 --api-key YOUR_USER_KEY +``` + +If you only need retrieval and ingestion, that is enough. + +If you also want the optional `query` tool, add a local `ov.conf` containing `vlm.api_base` and `vlm.model`: + +```bash +uv run server.py --url http://YOUR_SERVER:1933 --api-key YOUR_USER_KEY --config ./ov.conf +``` + +### Embedded/Local Mode + +If you want the bridge to open a local OpenViking workspace directly: + ```bash # Setup config cp ov.conf.example ov.conf # Edit ov.conf with your API keys -# Install dependencies -uv sync - -# Start the server (streamable HTTP on port 2033) +# Start the bridge (streamable HTTP on port 2033) uv run server.py ``` -The server will be available at `http://127.0.0.1:2033/mcp`. +The bridge will be available at `http://127.0.0.1:2033/mcp`. + +## Connect from Codex + +```bash +codex mcp add openviking --url http://127.0.0.1:2033/mcp +``` + +If you want to verify it was added: + +```bash +codex mcp list +``` ## Connect from Claude @@ -51,10 +96,15 @@ Or add to `.mcp.json`: ``` uv run server.py [OPTIONS] - --config PATH Config file path (default: ./ov.conf, env: OV_CONFIG) - --data PATH Data directory path (default: ./data, env: OV_DATA) - --host HOST Bind address (default: 127.0.0.1) - --port PORT Listen port (default: 2033, env: OV_PORT) + --config PATH Local ov.conf for optional query LLM config (default: ./ov.conf, env: OV_CONFIG) + --data PATH Local OpenViking data directory for embedded mode (default: ./data, env: OV_DATA) + --url URL Existing OpenViking HTTP server URL (env: OV_SERVER_URL) + --api-key KEY Existing OpenViking HTTP API key (env: OV_API_KEY) + --account ID Existing OpenViking account header for root-key access (env: OV_ACCOUNT) + --user ID Existing OpenViking user header for root-key access (env: OV_USER) + --agent-id ID Existing OpenViking agent header (env: OV_AGENT_ID) + --host HOST Bind address for the MCP bridge (default: 127.0.0.1) + --port PORT MCP bridge listen port (default: 2033, env: OV_PORT) --transport TYPE streamable-http | stdio (default: streamable-http) ``` @@ -64,3 +114,30 @@ uv run server.py [OPTIONS] npx @modelcontextprotocol/inspector # Connect to http://localhost:2033/mcp ``` + +## Memory Behavior + +This bridge does not automatically save Codex conversation turns into OpenViking. + +Claude's memory plugin does that through explicit lifecycle hooks (`SessionStart`, `Stop`, `SessionEnd`) that create an OpenViking session, append turns, and commit the session for memory extraction. Codex MCP integration does not provide that same hook flow here, so this bridge currently focuses on retrieval and resource ingestion. + +What it can do today is manual memory capture through MCP tools: + +1. Call `memory_start_session` +2. Call `memory_add_turn` for the important exchanges you want to keep +3. Call `memory_commit_session` when you want OpenViking to extract and index those memories + +Example flow: + +```text +memory_start_session() +→ {"session_id": "..."} + +memory_add_turn( + session_id="...", + user_message="Kalev prefers Codex + OpenViking over Claude-only workflows", + assistant_message="We set up an MCP bridge in front of the remote OpenViking server" +) + +memory_commit_session(session_id="...") +``` diff --git a/examples/mcp-query/server.py b/examples/mcp-query/server.py index fbc98e328..2c6ee9db9 100644 --- a/examples/mcp-query/server.py +++ b/examples/mcp-query/server.py @@ -18,7 +18,7 @@ import logging import os import sys -from pathlib import Path +from datetime import datetime from typing import Optional sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -26,9 +26,6 @@ from common.recipe import Recipe from mcp.server.fastmcp import FastMCP -import openviking as ov -from openviking_cli.utils.config.open_viking_config import OpenVikingConfig - logging.basicConfig(level=logging.INFO) logger = logging.getLogger("openviking-mcp") @@ -36,17 +33,51 @@ _recipe: Optional[Recipe] = None _config_path: str = "./ov.conf" _data_path: str = "./data" -_api_key: str = "" +_server_url: str = "" +_ov_api_key: str = "" +_ov_account: str = "" +_ov_user: str = "" +_ov_agent_id: str = "" +_llm_api_key: str = "" +_timeout: float = 60.0 _default_uri: str = "" +def _format_timestamp(raw_value: Optional[str]) -> Optional[str]: + """Format ISO-like timestamps into a user-friendly absolute time string.""" + if not raw_value: + return None + + try: + dt = datetime.fromisoformat(raw_value) + except ValueError: + return raw_value + + formatted = dt.strftime("%B %d, %Y %H:%M") + if dt.tzinfo is not None: + offset = dt.strftime("%z") + if offset == "+0000": + formatted += " UTC" + elif offset: + formatted += f" UTC{offset[:3]}:{offset[3:]}" + return formatted + + def _get_recipe() -> Recipe: """Get or create the Recipe (RAG pipeline) instance.""" global _recipe if _recipe is None: - _recipe = Recipe(config_path=_config_path, data_path=_data_path) - if _api_key: - _recipe.api_key = _api_key + _recipe = Recipe( + config_path=_config_path, + data_path=_data_path, + server_url=_server_url or None, + api_key=_ov_api_key or None, + account=_ov_account or None, + user=_ov_user or None, + agent_id=_ov_agent_id or None, + timeout=_timeout, + llm_api_key=_llm_api_key or None, + ) return _recipe @@ -55,9 +86,9 @@ def create_server(host: str = "127.0.0.1", port: int = 2033) -> FastMCP: mcp = FastMCP( name="openviking-mcp", instructions=( - "OpenViking MCP Server provides RAG (Retrieval-Augmented Generation) capabilities. " - "Use 'query' for full RAG answers, 'search' for semantic search only, " - "and 'add_resource' to ingest new documents." + "OpenViking MCP Server exposes OpenViking over MCP. " + "Use 'search' to retrieve context from OpenViking and 'add_resource' to ingest files, " + "directories, or URLs. Use 'query' only when this bridge also has local LLM config." ), host=host, port=port, @@ -72,7 +103,7 @@ async def query( temperature: float = 0.7, max_tokens: int = 2048, score_threshold: float = 0.2, - system_prompt: str = "", + system_prompt: Optional[str] = None, ) -> str: """ Ask a question and get an answer using RAG (Retrieval-Augmented Generation). @@ -91,6 +122,16 @@ async def query( def _query_sync(): recipe = _get_recipe() + if not recipe.query_ready: + return { + "answer": ( + "Query is not configured on this MCP bridge. " + "Provide a local ov.conf with vlm.api_base and vlm.model, " + "or use the search tool and let Codex synthesize the answer." + ), + "context": [], + "timings": {}, + } return recipe.query( user_query=question, search_top_k=top_k, @@ -127,7 +168,7 @@ async def search( query: str, top_k: int = 5, score_threshold: float = 0.2, - target_uri: str = "", + target_uri: Optional[str] = None, ) -> str: """ Search the OpenViking database for relevant content (no LLM generation). @@ -160,7 +201,17 @@ def _search_sync(): output_parts = [] for i, r in enumerate(results, 1): preview = r["content"][:500] + "..." if len(r["content"]) > 500 else r["content"] - output_parts.append(f"[{i}] {r['uri']} (score: {r['score']:.4f})\n{preview}") + timestamp_parts = [] + if r.get("updated_at"): + timestamp_parts.append(f"updated: {_format_timestamp(r['updated_at'])}") + if r.get("created_at"): + timestamp_parts.append(f"created: {_format_timestamp(r['created_at'])}") + timestamp_block = "" + if timestamp_parts: + timestamp_block = "\n" + "\n".join(timestamp_parts) + output_parts.append( + f"[{i}] {r['uri']} (score: {r['score']:.4f}){timestamp_block}\n{preview}" + ) return f"Found {len(results)} results:\n\n" + "\n\n".join(output_parts) @@ -176,52 +227,107 @@ async def add_resource(resource_path: str) -> str: Args: resource_path: Path to a local file/directory, or a URL to add. """ - config_path = _config_path - data_path = _data_path - def _add_sync(): - with open(config_path, "r") as f: - config_dict = json.load(f) - - config = OpenVikingConfig.from_dict(config_dict) - client = ov.SyncOpenViking(path=data_path, config=config) - - try: - client.initialize() - - path = resource_path - if not path.startswith("http"): - resolved = Path(path).expanduser() - if not resolved.exists(): - return f"Error: File not found: {resolved}" - path = str(resolved) - - result = client.add_resource(path=path) - - if result and "root_uri" in result: - root_uri = result["root_uri"] - client.wait_processed(timeout=300) - return f"Resource added and indexed: {root_uri}" - elif result and result.get("status") == "error": - errors = result.get("errors", [])[:3] - error_msg = "\n".join(f" - {e}" for e in errors) - return ( - f"Resource had parsing issues:\n{error_msg}\n" - "Some content may still be searchable." - ) - else: - return "Failed to add resource." - finally: - client.close() + recipe = _get_recipe() + return recipe.add_resource(resource_path) return await asyncio.to_thread(_add_sync) + @mcp.tool() + async def memory_start_session() -> dict: + """ + Create a new OpenViking session for manual memory capture. + + Call this once at the beginning of a task you want to remember, then use + `memory_add_turn` for the important exchanges and `memory_commit_session` + when you want OpenViking to extract memories. + """ + + def _start_sync(): + recipe = _get_recipe() + return recipe.create_memory_session() + + return await asyncio.to_thread(_start_sync) + + @mcp.tool() + async def memory_get_session(session_id: str) -> dict: + """ + Inspect an existing OpenViking memory session. + + Use this to recover the current message count or verify that a session id + is still valid before appending or committing. + """ + + def _get_sync(): + recipe = _get_recipe() + return recipe.get_memory_session(session_id) + + return await asyncio.to_thread(_get_sync) + + @mcp.tool() + async def memory_add_turn( + session_id: str, + user_message: Optional[str] = None, + assistant_message: Optional[str] = None, + note: Optional[str] = None, + ) -> dict: + """ + Append one important turn into an OpenViking memory session. + + Typical usage: + - Put the user's message in `user_message` + - Put the assistant's reply or short summary in `assistant_message` + - Put any extra context you want to preserve in `note` + """ + + def _add_turn_sync(): + recipe = _get_recipe() + return recipe.add_memory_turn( + session_id=session_id, + user_message=user_message, + assistant_message=assistant_message, + note=note, + ) + + return await asyncio.to_thread(_add_turn_sync) + + @mcp.tool() + async def memory_commit_session(session_id: str) -> dict: + """ + Commit an OpenViking session so memories are extracted and indexed. + + This is the manual equivalent of the Claude plugin's session-end commit step. + """ + + def _commit_sync(): + recipe = _get_recipe() + return recipe.commit_memory_session(session_id) + + return await asyncio.to_thread(_commit_sync) + + @mcp.tool() + async def memory_delete_session(session_id: str) -> dict: + """ + Delete an OpenViking memory session. + + Useful if you started a session by mistake and do not want to keep it. + """ + + def _delete_sync(): + recipe = _get_recipe() + return recipe.delete_memory_session(session_id) + + return await asyncio.to_thread(_delete_sync) + @mcp.resource("openviking://status") def server_status() -> str: """Get the current server status and configuration.""" info = { + "mode": "http" if _server_url else "local", "config_path": _config_path, "data_path": _data_path, + "server_url": _server_url or None, + "default_uri": _default_uri or None, "status": "running", } return json.dumps(info, indent=2) @@ -235,28 +341,40 @@ def parse_args(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - # Start with defaults + # Start in embedded/local mode uv run server.py - # Custom config and port - uv run server.py --config ./ov.conf --data ./data --port 9000 + # Bridge to an existing remote OpenViking HTTP server + uv run server.py --url http://192.168.1.50:1933 --api-key sk-xxx + + # Bridge to a remote OpenViking HTTP server with root-key tenant headers + uv run server.py --url http://192.168.1.50:1933 --api-key root-key --account acme --user alice + + # Remote OpenViking for search/add-resource, but local ov.conf for the optional query tool + uv run server.py --url http://192.168.1.50:1933 --config ./ov.conf # Use stdio transport (for Claude Desktop integration) uv run server.py --transport stdio - # Connect from Claude CLI (use 127.0.0.1 instead of localhost for Windows compatibility) - claude mcp add --transport http openviking http://127.0.0.1:2033/mcp + # Connect from Codex + codex mcp add openviking --url http://127.0.0.1:2033/mcp - # With API key and default search scope - uv run server.py --api-key sk-xxx --default-uri viking://user/memories + # With default search scope + uv run server.py --url http://192.168.1.50:1933 --default-uri viking://user/memories Environment variables: - OV_CONFIG Path to config file (default: ./ov.conf) - OV_DATA Path to data directory (default: ./data) - OV_PORT Server port (default: 2033) - OV_API_KEY API key for OpenViking server authentication - OV_DEFAULT_URI Default target URI for search scoping - OV_DEBUG Enable debug logging (set to 1) + OV_CONFIG Path to local ov.conf for the optional query tool (default: ./ov.conf) + OV_DATA Path to local OpenViking data directory (default: ./data) + OV_PORT MCP bridge port (default: 2033) + OV_SERVER_URL Remote OpenViking HTTP server URL + OV_API_KEY Remote OpenViking HTTP API key + OV_ACCOUNT Remote OpenViking account header (root-key access only) + OV_USER Remote OpenViking user header (root-key access only) + OV_AGENT_ID Remote OpenViking agent header + OV_LLM_API_KEY Override the local query LLM API key from ov.conf + OV_DEFAULT_URI Default target URI for search scoping + OV_TIMEOUT Timeout in seconds for OpenViking and query LLM calls + OV_DEBUG Enable debug logging (set to 1) """, ) parser.add_argument( @@ -271,6 +389,12 @@ def parse_args(): default=os.getenv("OV_DATA", "./data"), help="Path to data directory (default: ./data)", ) + parser.add_argument( + "--url", + type=str, + default=os.getenv("OV_SERVER_URL", ""), + help="Remote OpenViking HTTP server URL (default: local embedded mode)", + ) parser.add_argument( "--host", type=str, @@ -294,7 +418,37 @@ def parse_args(): "--api-key", type=str, default=os.getenv("OV_API_KEY", ""), - help="API key for OpenViking server authentication (default: $OV_API_KEY)", + help="API key for remote OpenViking HTTP authentication (default: $OV_API_KEY)", + ) + parser.add_argument( + "--account", + type=str, + default=os.getenv("OV_ACCOUNT", ""), + help="Remote OpenViking account header (needed with root key access)", + ) + parser.add_argument( + "--user", + type=str, + default=os.getenv("OV_USER", ""), + help="Remote OpenViking user header (needed with root key access)", + ) + parser.add_argument( + "--agent-id", + type=str, + default=os.getenv("OV_AGENT_ID", ""), + help="Remote OpenViking agent header", + ) + parser.add_argument( + "--llm-api-key", + type=str, + default=os.getenv("OV_LLM_API_KEY", ""), + help="Override the local query LLM API key from ov.conf", + ) + parser.add_argument( + "--timeout", + type=float, + default=float(os.getenv("OV_TIMEOUT", "60")), + help="Timeout in seconds for OpenViking and query LLM calls", ) parser.add_argument( "--default-uri", @@ -308,18 +462,36 @@ def parse_args(): def main(): args = parse_args() - global _config_path, _data_path, _api_key, _default_uri + global _config_path, _data_path, _server_url, _ov_api_key, _ov_account, _ov_user + global _ov_agent_id, _llm_api_key, _timeout, _default_uri _config_path = args.config _data_path = args.data - _api_key = args.api_key + _server_url = args.url + _ov_api_key = args.api_key + _ov_account = args.account + _ov_user = args.user + _ov_agent_id = args.agent_id + _llm_api_key = args.llm_api_key + _timeout = args.timeout _default_uri = args.default_uri if os.getenv("OV_DEBUG") == "1": logging.getLogger().setLevel(logging.DEBUG) + if not _server_url and not os.path.exists(_config_path): + raise SystemExit( + f"Config file not found: {_config_path}. " + "Create a local ov.conf or pass --url to bridge to a remote OpenViking HTTP server." + ) + logger.info("OpenViking MCP Server starting") + logger.info(f" mode: {'http-bridge' if _server_url else 'local'}") logger.info(f" config: {_config_path}") logger.info(f" data: {_data_path}") + if _server_url: + logger.info(f" ov url: {_server_url}") + if _ov_account or _ov_user: + logger.info(f" tenant: account={_ov_account or '-'} user={_ov_user or '-'}") logger.info(f" transport: {args.transport}") mcp = create_server(host=args.host, port=args.port) diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py index b60d612af..d63468f91 100644 --- a/openviking/retrieve/hierarchical_retriever.py +++ b/openviking/retrieve/hierarchical_retriever.py @@ -564,6 +564,8 @@ async def _convert_to_matched_contexts( abstract=c.get("abstract", ""), category=c.get("category", ""), score=final_score, + created_at=c.get("created_at"), + updated_at=c.get("updated_at"), relations=relations, ) ) diff --git a/openviking_cli/retrieve/types.py b/openviking_cli/retrieve/types.py index f1ca51060..a9dcc4664 100644 --- a/openviking_cli/retrieve/types.py +++ b/openviking_cli/retrieve/types.py @@ -289,6 +289,8 @@ class MatchedContext: category: str = "" score: float = 0.0 match_reason: str = "" + created_at: Optional[str] = None + updated_at: Optional[str] = None relations: List[RelatedContext] = field(default_factory=list) @@ -371,6 +373,8 @@ def _context_to_dict(self, ctx: MatchedContext) -> Dict[str, Any]: "score": ctx.score, "category": ctx.category, "match_reason": ctx.match_reason, + "created_at": ctx.created_at, + "updated_at": ctx.updated_at, "relations": [{"uri": r.uri, "abstract": r.abstract} for r in ctx.relations], "abstract": ctx.abstract, "overview": ctx.overview, @@ -399,6 +403,8 @@ def _parse_context(d: Dict[str, Any]) -> MatchedContext: category=d.get("category", ""), score=d.get("score", 0.0), match_reason=d.get("match_reason", ""), + created_at=d.get("created_at"), + updated_at=d.get("updated_at"), relations=[ RelatedContext(uri=r.get("uri", ""), abstract=r.get("abstract", "")) for r in d.get("relations", [])