diff --git a/.env.example b/.env.example index 40ea8b9..a6d8d15 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,41 @@ # ============================================================================ # Dexiask environment — copy to `.env` and fill in the secrets. # cp .env.example .env -# All secrets are brought by you; nothing is baked into images. +# All secrets are brought by you. No API keys at all? Enable local mode below. # ============================================================================ -# --- Secrets (required) -------------------------------------------------- +# --- Local mode (run with NO API keys) ------------------------------------- +# Uncomment to add an Ollama sidecar that serves a small local text model and +# local code embeddings. Both models are baked into the sidecar image at build +# time (first build downloads ~2 GB; nothing is downloaded after the stack is +# up). With no keys set, the engine and indexer fall back to it automatically; +# real keys, when present, always win. Runs CPU-only on a regular laptop +# (~3 GB RAM); expect simpler answers than Claude. Switching between local and +# hosted embeddings requires a re-index (`make clean` or per-repo reindex): +# same dimension, different vector space. +#COMPOSE_PROFILES=local +# Models baked into the sidecar image (build args). Pick a non-reasoning, +# tool-capable text model; bigger = better answers but more RAM/slower on CPU +# (e.g. qwen2.5:7b ~4.7 GB). The embedding model must produce +# DEXIASK_EMBEDDING_DIM-sized vectors (qwen3-embedding:0.6b → 1024). +#DEXIASK_LOCAL_TEXT_MODEL=qwen2.5:1.5b +#DEXIASK_LOCAL_EMBED_MODEL=qwen3-embedding:0.6b +# Tuning baked into the text model at build time (build args — rebuild to +# apply). Context: Ollama's 4096 default would truncate the agent's system +# prompt; larger = more RAM. Threads: llama.cpp defaults to every core, which +# on laptops is dramatically SLOWER than a few pinned threads; raise only on +# real many-core servers. +#DEXIASK_OLLAMA_CONTEXT_LENGTH=16384 +#DEXIASK_OLLAMA_THREADS=4 +# Remote MCP tools exposed to the local model: '*' (default) = every tool the +# indexer/memory servers offer. If a small model stops calling tools and +# answers from thin air, narrow this to a short list — fewer tools are easier +# for small models to follow: +#DEXIASK_LOCAL_REMOTE_TOOLS=semantic_search,get_chunk,read_range,get_overview,memory_search +# On a CPU-only box, consider disabling the hourly dream consolidation: +#DEXIASK_DREAM_INTERVAL=0 + +# --- Secrets (optional in local mode, required otherwise) ------------------ # Anthropic API key used by the Claude engine. This may also be a key for an # Anthropic-compatible gateway (see ANTHROPIC_BASE_URL below). ANTHROPIC_API_KEY= @@ -16,6 +47,9 @@ ANTHROPIC_API_KEY= ANTHROPIC_BASE_URL= # Voyage AI key used by the indexer for code embeddings. Get one at voyageai.com. VOYAGE_API_KEY= +# Optional alternative embedding key; the indexer picks the first configured +# provider: Voyage → OpenAI → local Ollama sidecar. +OPENAI_API_KEY= # --- Slack bot (optional) ------------------------------------------------ # Leave blank to disable the Slack bot. Both must be set to enable it. diff --git a/CLAUDE.md b/CLAUDE.md index 49ed355..0acc392 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -139,10 +139,25 @@ manages them via the `/api/mcp/*` BFF proxy. ## Environment Copy `.env.example` to `.env`. All secrets come from env (`ANTHROPIC_API_KEY`, -`VOYAGE_API_KEY`, `SLACK_*`); nothing is baked into images. Service URLs use +`VOYAGE_API_KEY`, `SLACK_*`); no secrets are baked into images. Service URLs use compose-internal DNS (`http://engine:8080`, `http://indexer:8080`, `http://memory:8080`, `http://qdrant:6333`, `postgres:5432`). +**Local mode (no API keys)**: `COMPOSE_PROFILES=local` in `.env` adds an `ollama` +sidecar whose text + embedding models (`DEXIASK_LOCAL_TEXT_MODEL` / +`DEXIASK_LOCAL_EMBED_MODEL`, defaults `qwen2.5:1.5b` / `qwen3-embedding:0.6b`) are +deliberately **baked into the image at build time** — nothing downloads after the +stack is up. The profile injects `CLAUDE_ENGINE_LOCAL_BASE_URL` / +`DEXIASK_OLLAMA_BASE_URL` (= `http://ollama:11434`); with no API key the engine +dispatches jobs to `engine_core.local_runtime.LocalOllamaRuntime` — a compact +tool-calling loop over Ollama's native `/api/chat` (role prompt + workspace tools + +the `local_remote_tools` MCP allowlist; no Claude CLI, whose scaffolding is far too +many prompt tokens for a small CPU model) with its own FS session store — overriding +the Job's model with the local one (dream jobs included). The indexer's `auto` +embedding provider resolves voyage → openai → ollama. Real keys always win. Local +and hosted embeddings share a dim (1024) but not a vector space: switching requires +a re-index. + Feature-gating env (all optional; each degrades gracefully when unset): - **Auth**: `DEXIASK_GITHUB_CLIENT_ID`/`_SECRET` (empty → dev-fallback, no login), `DEXIASK_OAUTH_CALLBACK_URL`, `DEXIASK_WEB_BASE_URL`, `DEXIASK_SESSION_SECRET`, diff --git a/Makefile b/Makefile index b724952..f5938c5 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ .DEFAULT_GOAL := help COMPOSE := docker compose -.PHONY: help up down build logs ps restart test test-backend test-memory test-engine test-indexer test-web lint fmt clean +.PHONY: help up up-local down build logs ps restart test test-backend test-memory test-engine test-indexer test-web lint fmt clean help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ @@ -12,6 +12,10 @@ up: ## Build (if needed) and start the whole stack $(COMPOSE) up --build -d @echo "web → http://localhost:$${DEXIASK_WEB_PORT:-25051}" +up-local: ## Start with the local no-API-key Ollama sidecar (same as COMPOSE_PROFILES=local in .env) + COMPOSE_PROFILES=local $(COMPOSE) up --build -d + @echo "web → http://localhost:$${DEXIASK_WEB_PORT:-25051}" + down: ## Stop the stack $(COMPOSE) down diff --git a/README.md b/README.md index 2165efb..b7bbc01 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,8 @@ it directly via semantic + lexical search. - **memory** — Go. FS-backed user/repo/global memory exposed as an MCP server, with a periodic "dream" consolidation judge. No database. - **qdrant / postgres** — vector store and relational store. +- **ollama** *(optional, `COMPOSE_PROFILES=local`)* — local text + embedding models + baked into the image, so everything runs with no API keys. One **shared `/workspace` mount** is the codebase the agent reads and the indexer indexes (read-only for the indexer). @@ -108,7 +110,8 @@ indexes (read-only for the indexer). ## Quickstart **Prerequisites:** Docker + Docker Compose, an [Anthropic API key](https://console.anthropic.com/), -and a [Voyage AI key](https://www.voyageai.com/) (for code embeddings). +and a [Voyage AI key](https://www.voyageai.com/) (for code embeddings) — or **no keys +at all** with [local mode](#local-mode-no-api-keys) below. ```bash git clone dexiask && cd dexiask @@ -121,6 +124,48 @@ make up # or: docker compose up --build -d Open **http://localhost:25051** and start chatting. `make logs` to tail, `make down` to stop, `make clean` to wipe the DB + index. +### Local mode (no API keys) + +No Anthropic or Voyage key? Enable the **Ollama sidecar** and run entirely locally: + +```bash +cp .env.example .env +# → edit .env: uncomment COMPOSE_PROFILES=local (leave the key vars blank) +make up +``` + +The sidecar serves a small local text model (`qwen2.5:1.5b`) and local code +embeddings (`qwen3-embedding:0.6b`). Both models are **baked into the sidecar +image at build time** — the first build downloads ~2 GB; nothing is downloaded +after the stack is up. With no keys configured the engine and indexer fall back +to the sidecar automatically (the engine switches to a compact local agent loop +sized for small models — first answer in ~30s on a laptop CPU, follow-ups in +seconds); if you later add real keys they take over (hosted keys always win). + +Worth knowing: + +- **Quality/speed**: a small local model is noticeably less capable than Claude — + fine for exploring a codebase, not a Claude replacement. For better answers at + a higher resource cost, set a bigger **non-reasoning** model, e.g. + `DEXIASK_LOCAL_TEXT_MODEL=qwen2.5:7b` (~4.7 GB). Avoid reasoning models + (qwen3, deepseek-r1): they burn the whole turn "thinking" on CPU. +- **Resources**: runs CPU-only on a regular laptop — ~4 GB free disk for the + image and ~3 GB RAM while running. The sidecar pins generation to 4 threads + (`DEXIASK_OLLAMA_THREADS`) — counter-intuitively much faster than all cores on + laptops. On an NVIDIA GPU host, uncomment the GPU block on the `ollama` + service in `docker-compose.yml` for faster responses. +- **Tools**: the local agent gets every indexer/memory MCP tool by default + (`DEXIASK_LOCAL_REMOTE_TOOLS=*`). If a small model stops calling tools and + answers from thin air, narrow it to a short list (e.g. + `semantic_search,get_chunk,read_range`) — fewer tools are easier to follow. +- **CPU-only boxes**: set `DEXIASK_DREAM_INTERVAL=0` to skip the hourly memory + consolidation model runs. +- **Re-index on switch**: local and hosted embeddings share a dimension but not a + vector space — switching providers requires `make clean` (or a per-repo reindex). +- Models are overridable via `DEXIASK_LOCAL_TEXT_MODEL` / `DEXIASK_LOCAL_EMBED_MODEL` + (image build args; the embedding model must produce `DEXIASK_EMBEDDING_DIM`-sized + vectors). + ## Indexing a repository The indexer indexes the **default branch** of any git repo under your mounted @@ -214,8 +259,10 @@ Everything is env-driven — see `.env.example` for the full list. The essential | Variable | Purpose | |---|---| -| `ANTHROPIC_API_KEY` | Claude engine credential (required) | -| `VOYAGE_API_KEY` | Indexer embedding credential (required) | +| `COMPOSE_PROFILES=local` | Enable the Ollama sidecar — run with **no API keys** (see [Local mode](#local-mode-no-api-keys)) | +| `ANTHROPIC_API_KEY` | Claude engine credential (required unless local mode) | +| `VOYAGE_API_KEY` | Indexer embedding credential (required unless local mode) | +| `DEXIASK_LOCAL_TEXT_MODEL` / `_LOCAL_EMBED_MODEL` | Models baked into the local sidecar image (defaults `qwen2.5:1.5b` / `qwen3-embedding:0.6b`) | | `DEXIASK_MODEL` | Claude model for ask mode (default `claude-sonnet-5`) | | `DEXIASK_WORKSPACE_PATH` | Host codebase mounted at `/workspace` | | `DEXIASK_SESSION_SECRET` / `_TOKEN_ENC_KEY` | Session signing + GitHub-token encryption — enables auth (token login) | diff --git a/docker-compose.yml b/docker-compose.yml index 4739664..aaea13a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,15 +35,62 @@ services: timeout: 5s retries: 10 + # Local no-API-key mode: `COMPOSE_PROFILES=local` in .env enables this sidecar + # and the fallback env below. Models are baked into the image at build time + # (build args) — nothing is downloaded after the stack is up. + ollama: + build: + context: ./ollama + args: + TEXT_MODEL: ${DEXIASK_LOCAL_TEXT_MODEL:-qwen2.5:1.5b} + EMBED_MODEL: ${DEXIASK_LOCAL_EMBED_MODEL:-qwen3-embedding:0.6b} + # Tuning baked into the TEXT model only (not global env), so the tiny + # embedding model keeps its small default context. See ollama/Dockerfile + # for why (KV-cache size; thread-contention slowdown on laptops). + TEXT_CONTEXT: ${DEXIASK_OLLAMA_CONTEXT_LENGTH:-16384} + TEXT_THREADS: ${DEXIASK_OLLAMA_THREADS:-4} + profiles: ["local"] + environment: + OLLAMA_MODELS: /models + # Keep models resident (text + embed loaded side by side). + OLLAMA_KEEP_ALIVE: -1 + OLLAMA_MAX_LOADED_MODELS: 2 + # One slot → the llama.cpp prefix cache holds the conversation prefix + # between turns, so turn N only evaluates the delta. + OLLAMA_NUM_PARALLEL: 1 + healthcheck: + test: ["CMD", "ollama", "list"] + interval: 10s + timeout: 5s + retries: 10 + # On an NVIDIA GPU host, uncomment to offload inference: + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: all + # capabilities: [gpu] + engine: build: context: ./engine environment: - ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:?set ANTHROPIC_API_KEY in .env} + # Optional when the `local` profile is enabled — the engine then falls + # back to the Ollama sidecar. Required for Claude-quality answers. + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} # Optional: point the Claude SDK at an Anthropic-compatible gateway # (e.g. OpenRouter → https://openrouter.ai/api). Blank = api.anthropic.com. ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-} CLAUDE_ENGINE_MODEL: ${DEXIASK_MODEL:-claude-sonnet-5} + # Local fallback (set only when the `local` profile is on): with no API + # key the engine runs jobs against the Ollama sidecar's Anthropic-compat + # API using this model instead of erroring. A real key always wins. + CLAUDE_ENGINE_LOCAL_BASE_URL: ${COMPOSE_PROFILES:+http://ollama:11434} + CLAUDE_ENGINE_LOCAL_MODEL: ${DEXIASK_LOCAL_TEXT_MODEL:-qwen2.5:1.5b} + # Remote MCP tools exposed to the local model: '*' = all, or a comma + # allowlist — small models often follow tools better with fewer of them. + CLAUDE_ENGINE_LOCAL_REMOTE_TOOLS: ${DEXIASK_LOCAL_REMOTE_TOOLS:-*} # No OTel collector ships with the stack; disable the exporter to keep logs # clean. Set to "false" and add a collector to enable tracing. OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-true} @@ -62,10 +109,16 @@ services: build: context: ./indexer environment: - DEXIASK_VOYAGE_API_KEY: ${VOYAGE_API_KEY:?set VOYAGE_API_KEY in .env} + # Optional when the `local` profile is enabled — the `auto` provider then + # falls back to the Ollama sidecar for embeddings. A real key always wins. + DEXIASK_VOYAGE_API_KEY: ${VOYAGE_API_KEY:-} DEXIASK_QDRANT_URL: ${DEXIASK_QDRANT_URL:-http://qdrant:6333} + DEXIASK_EMBEDDING_PROVIDER: ${DEXIASK_EMBEDDING_PROVIDER:-auto} DEXIASK_EMBEDDING_MODEL: ${DEXIASK_EMBEDDING_MODEL:-voyage-code-3} DEXIASK_EMBEDDING_DIM: ${DEXIASK_EMBEDDING_DIM:-1024} + DEXIASK_OPENAI_API_KEY: ${OPENAI_API_KEY:-} + DEXIASK_OLLAMA_BASE_URL: ${COMPOSE_PROFILES:+http://ollama:11434} + DEXIASK_OLLAMA_EMBEDDING_MODEL: ${DEXIASK_LOCAL_EMBED_MODEL:-qwen3-embedding:0.6b} # Generated domain-knowledge docs (LLM), embedded as content_type=doc and # searchable alongside code. Off by default; needs an Anthropic key. DEXIASK_ENABLE_DOMAIN_DOCS: ${DEXIASK_ENABLE_DOMAIN_DOCS:-false} diff --git a/engine/.env.example b/engine/.env.example index 460c44f..e18a098 100644 --- a/engine/.env.example +++ b/engine/.env.example @@ -11,6 +11,16 @@ CLAUDE_ENGINE_MODEL=claude-sonnet-5 # Anthropic-compatible gateway base URL (e.g. a local proxy). Empty = default. ANTHROPIC_BASE_URL= +# ── Local model fallback (no API key) ─────────────────────────────────────── +# With no key configured anywhere, jobs run on the compact local runtime against +# this Ollama server (the compose `local` profile's sidecar) with the local +# model replacing the Job's. Empty = fallback unavailable (missing key errors). +# CLAUDE_ENGINE_LOCAL_BASE_URL=http://ollama:11434 +# CLAUDE_ENGINE_LOCAL_MODEL=qwen2.5:1.5b +# Remote MCP tools the local runtime exposes (small models need few tools): +# CLAUDE_ENGINE_LOCAL_REMOTE_TOOLS=* # or a comma allowlist, e.g. semantic_search,get_chunk +# CLAUDE_ENGINE_LOCAL_REQUEST_TIMEOUT_S=300 + # HTTP server bind (container listens on 8080). CLAUDE_ENGINE_SERVER_HOST=0.0.0.0 CLAUDE_ENGINE_SERVER_PORT=8080 diff --git a/engine/engine_core/capabilities/mcp_router.py b/engine/engine_core/capabilities/mcp_router.py index 08a045b..fc630ee 100644 --- a/engine/engine_core/capabilities/mcp_router.py +++ b/engine/engine_core/capabilities/mcp_router.py @@ -89,6 +89,14 @@ async def _connect(self, name: str) -> AsyncIterator[ClientSession]: await session.initialize() yield session + async def catalog(self, name: str) -> list[dict[str, Any]]: + """The cached tool list for *name* (fetched on first use). + + Public for consumers that need exact tool names/schemas rather than a + capability search (the local runtime's tool allowlist). + """ + return await self._ensure_catalog(name) + async def _ensure_catalog(self, name: str) -> list[dict[str, Any]]: """Fetch + cache the tool list for *name* (once per run).""" if name in self._catalog: diff --git a/engine/engine_core/local_runtime.py b/engine/engine_core/local_runtime.py new file mode 100644 index 0000000..a851a39 --- /dev/null +++ b/engine/engine_core/local_runtime.py @@ -0,0 +1,266 @@ +""" +Local-model agent loop — the no-API-key fallback runtime. + +Runs a compact tool-calling loop directly against a local Ollama server's +native ``/api/chat`` instead of driving the full Claude Code CLI. The CLI's +scaffolding (its own system prompt, every MCP tool schema, the skill packs) +adds tens of thousands of prompt tokens per turn — fine for hosted Claude, +but a small local model on laptop CPU spends minutes just *evaluating* that +prompt. This loop keeps the prompt to the Job's role prompt plus a handful of +tools, so small models start answering in seconds. + +Kept deliberately small: + - Tools: the workspace descriptors (Read/Glob/Grep) plus an allowlist of + remote MCP tools (``settings.local_remote_tools``, e.g. semantic_search) + resolved via the shared ``RemoteMcpRouter``. + - No skills addendum, no interactive tools, no deferred-MCP router — a + small model cannot use them effectively and they bloat the context. + - Session continuity: the orchestrator sends only the new user message plus + a sessionId (no history replay), so this runtime persists its own + user/assistant history as JSON under the session store path. +""" +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Any + +import httpx + +from .capabilities.mcp_router import RemoteMcpRouter +from .models.events import ( + AgentStepEvent, + ResultEvent, + TextDeltaEvent, + TextStartEvent, + TextStopEvent, + ToolInputDoneEvent, + ToolResultEvent, + ToolStartEvent, + Usage, + log, +) +from .runtime import AgentRuntime, RunContext +from .tools.descriptors import ToolDescriptor, invoke_safely + +# Cap on a single tool result fed back into the model, protecting the small +# local context window from one oversized file read or search result. +_MAX_TOOL_RESULT_CHARS = 8_000 + + +def _content_text(payload: dict[str, Any]) -> str: + """Flatten an SDK-MCP content payload ({"content": [blocks]}) to text.""" + parts = [b.get("text", "") for b in payload.get("content", []) if b.get("type") == "text"] + text = "\n".join(p for p in parts if p) or "(no content)" + if len(text) > _MAX_TOOL_RESULT_CHARS: + text = text[:_MAX_TOOL_RESULT_CHARS] + "\n[... truncated]" + return text + + +def _attachment_note(message: Any) -> str: + """Render a message's attachments as a plain-text note (no vision locally).""" + notes = [ + f"[Attached file: {att.filename} — read it at {att.path}]" + for att in message.attachments + ] + return "\n".join(notes) + + +class _SessionStore: + """User/assistant history persisted per session id (JSON on the FS).""" + + def __init__(self, root: str) -> None: + self._dir = Path(root) / "local-sessions" + + def load(self, session_id: str) -> list[dict[str, str]]: + path = self._dir / f"{session_id}.json" + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return [] + + def save(self, session_id: str, history: list[dict[str, str]]) -> None: + try: + self._dir.mkdir(parents=True, exist_ok=True) + (self._dir / f"{session_id}.json").write_text( + json.dumps(history), encoding="utf-8" + ) + except OSError as exc: # pragma: no cover - defensive + log(f"local runtime: could not persist session {session_id!r}: {exc}") + + +class LocalOllamaRuntime(AgentRuntime): + """Minimal agent loop over a local Ollama server (native /api/chat).""" + + def name(self) -> str: + return "local" + + def credential_env_names(self) -> tuple[str, str]: + return ("OLLAMA_API_KEY", "OLLAMA_BASE_URL") + + def supports_interactive(self) -> bool: + return False + + # ── Tool assembly ────────────────────────────────────────────────────── + + @staticmethod + def _tool_def(name: str, description: str, schema: dict[str, Any]) -> dict[str, Any]: + return { + "type": "function", + "function": {"name": name, "description": description, "parameters": schema}, + } + + async def _assemble_tools( + self, ctx: RunContext + ) -> tuple[list[dict[str, Any]], dict[str, ToolDescriptor], dict[str, str], RemoteMcpRouter | None]: + """Build the Ollama tool definitions plus dispatch maps. + + Returns (tool_defs, builtin_by_name, remote_tool→server, router). + """ + defs: list[dict[str, Any]] = [] + builtin: dict[str, ToolDescriptor] = {} + for d in ctx.builtin_tools.workspace: + builtin[d.name] = d + defs.append(self._tool_def(d.name, d.description, d.input_schema)) + + remote: dict[str, str] = {} + router: RemoteMcpRouter | None = None + allowlist = { + t.strip() for t in ctx.settings.local_remote_tools.split(",") if t.strip() + } + expose_all = "*" in allowlist + if allowlist and ctx.remote_native: + router = RemoteMcpRouter(ctx.remote_native, ctx.settings) + for server in router.server_names: + try: + tools = await router.catalog(server) + except Exception as e: # a down server must not sink the turn + log(f"local runtime: tool catalog failed for {server!r}: {e}") + continue + for t in tools: + if (expose_all or t["name"] in allowlist) and t["name"] not in remote: + remote[t["name"]] = server + defs.append( + self._tool_def(t["name"], t["description"], t["input_schema"]) + ) + return defs, builtin, remote, router + + # ── The loop ─────────────────────────────────────────────────────────── + + async def run(self, ctx: RunContext) -> str | None: + settings = ctx.settings + base = settings.local_base_url.rstrip("/") + session_id = ctx.session_id or str(uuid.uuid4()) + store = _SessionStore(ctx.session_store_path or settings.session_store_path) + history = store.load(session_id) if ctx.session_id else [] + + # The Job's role prompt (+ memory digest) only — deliberately not + # ctx.system_prompt, which appends the skill packs. + messages: list[dict[str, Any]] = [ + {"role": "system", "content": ctx.job.system_prompt} + ] + messages.extend(history) + new_turns: list[dict[str, str]] = [] + for m in ctx.messages: + content = m.content + note = _attachment_note(m) + if note: + content = f"{content}\n{note}".strip() + entry = {"role": m.role, "content": content} + messages.append(entry) + new_turns.append(entry) + + tool_defs, builtin, remote, router = await self._assemble_tools(ctx) + log( + f"local runtime: model={ctx.model!r} at {base!r} " + f"tools={[t['function']['name'] for t in tool_defs]!r}" + ) + + usage = Usage() + answer_parts: list[str] = [] + turns = 0 + async with httpx.AsyncClient(timeout=settings.local_request_timeout_s) as client: + for turn in range(1, ctx.max_turns + 1): + turns = turn + payload: dict[str, Any] = { + "model": ctx.model, + "messages": messages, + "stream": True, + "options": {"num_predict": ctx.max_tokens}, + } + if tool_defs: + payload["tools"] = tool_defs + + content_parts: list[str] = [] + tool_calls: list[dict[str, Any]] = [] + text_id: str | None = None + async with client.stream("POST", f"{base}/api/chat", json=payload) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.strip(): + continue + chunk = json.loads(line) + msg = chunk.get("message") or {} + delta = msg.get("content") or "" + if delta: + if text_id is None: + text_id = f"text-{turn}" + ctx.emit(TextStartEvent(id=text_id)) + ctx.emit(TextDeltaEvent(id=text_id, text=delta)) + content_parts.append(delta) + tool_calls.extend(msg.get("tool_calls") or []) + if chunk.get("done"): + usage.input_tokens += chunk.get("prompt_eval_count", 0) + usage.output_tokens += chunk.get("eval_count", 0) + if text_id is not None: + ctx.emit(TextStopEvent(id=text_id)) + + content = "".join(content_parts) + if content: + answer_parts.append(content) + if not tool_calls: + break + + assistant_msg: dict[str, Any] = {"role": "assistant", "content": content} + assistant_msg["tool_calls"] = tool_calls + messages.append(assistant_msg) + for i, call in enumerate(tool_calls): + fn = call.get("function") or {} + name = fn.get("name", "") + args = fn.get("arguments") or {} + if isinstance(args, str): # some models emit JSON strings + try: + args = json.loads(args) + except ValueError: + args = {} + tool_id = f"tool-{turn}-{i}" + ctx.emit(ToolStartEvent(id=tool_id, name=name)) + ctx.emit(ToolInputDoneEvent(id=tool_id, name=name, input=args)) + ctx.emit(AgentStepEvent(step=turn, tool=name)) + if name in builtin: + result = await invoke_safely(builtin[name], args) + elif name in remote and router is not None: + result = await router.proxy_call(remote[name], name, args) + else: + result = ToolDescriptor.text_result( + f"Error: unknown tool {name!r}", is_error=True + ) + text = _content_text(result) + ctx.emit(ToolResultEvent(id=tool_id, result=text)) + messages.append({"role": "tool", "tool_name": name, "content": text}) + + # Persist only the user/assistant exchange (tool traffic stays + # per-turn) so future prompts stay small. + answer = "\n".join(answer_parts) + store.save(session_id, history + new_turns + [{"role": "assistant", "content": answer}]) + + ctx.emit( + ResultEvent( + model=ctx.model, + session_id=session_id, + usage=usage, + num_turns=turns, + ) + ) + return session_id diff --git a/engine/engine_core/models/settings.py b/engine/engine_core/models/settings.py index 0e91c93..d9ccd18 100644 --- a/engine/engine_core/models/settings.py +++ b/engine/engine_core/models/settings.py @@ -7,11 +7,11 @@ its own provider credentials (``ANTHROPIC_API_KEY`` for Claude) and set its own ``env_prefix`` and ``default_model``. -Provider credentials are supplied per-request on the Job (``job.api_key`` / -``job.base_url``), resolved by the Go orchestrator from the workspace's -UI-configured agent settings — the runner never falls back to engine ``.env``. -The ``engine_api_key`` / ``engine_base_url`` accessors remain only for tests and -diagnostics that introspect a configured key. +Provider credentials come from the Job (``job.api_key`` / ``job.base_url``) +when the orchestrator supplies them, else from the engine env via the +``engine_api_key`` / ``engine_base_url`` accessors. When neither yields a key, +the runner falls back to the local model server (``local_base_url`` / +``local_model`` — the docker-compose Ollama sidecar) if one is configured. Use a per-engine cached accessor (``get_settings``) so tests can monkeypatch the singleton; ``default_settings()`` here returns a bare base instance for the @@ -148,19 +148,47 @@ class BaseEngineSettings(BaseSettings): ), ) + # ── Local model fallback (no-API-key mode) ───────────────────────────── + # When neither the Job nor the env yields an API key, the runner runs jobs + # against this provider-compatible local server (the docker-compose Ollama + # sidecar) instead of erroring. Empty base URL = fallback unavailable. + local_base_url: str = Field( + default="", + description=( + "Base URL of a local provider-compatible model server " + "(e.g. http://ollama:11434). Used only when no API key is configured." + ), + ) + local_model: str = Field( + default="", + description="Model served locally; replaces the Job's model on fallback.", + ) + local_remote_tools: str = Field( + default="*", + description=( + "Comma-separated allowlist of remote MCP tool names the local " + "runtime exposes to the model, or '*' for every tool the attached " + "servers offer. Narrow it (e.g. 'semantic_search,get_chunk') to " + "shrink the prompt when a small model gets confused by many tools." + ), + ) + local_request_timeout_s: float = Field( + default=300.0, + gt=0, + description="Per-request timeout (seconds) for local model calls.", + ) + # ── Provider credentials (subclasses override) ───────────────────────── - # NOTE: the runner no longer reads these — credentials come ONLY from the - # Job (job.api_key / job.base_url). These accessors are retained for tests - # and diagnostics that introspect a configured key; engines must NOT use - # them as a runtime credential fallback. + # Env-configured credentials, used when the Job carries none (the OSS + # single-user path). @property def engine_api_key(self) -> str: - """Provider API key, if any is configured in env. Diagnostics only — not a runtime fallback.""" + """Provider API key, if any is configured in env.""" return "" @property def engine_base_url(self) -> str: - """Provider base URL, if any is configured in env. Diagnostics only — not a runtime fallback.""" + """Provider base URL, if any is configured in env.""" return "" diff --git a/engine/engine_core/runner.py b/engine/engine_core/runner.py index 4dd483e..4502ec9 100644 --- a/engine/engine_core/runner.py +++ b/engine/engine_core/runner.py @@ -49,11 +49,6 @@ async def run( Emits an ``ErrorEvent`` and returns ``None`` on unrecoverable setup errors. """ model = job.resolved_model(settings.default_model) - log( - f"Starting. runtime={runtime.name()!r} role={job.role.value!r} model={model!r} " - f"allowed_tools={job.allowed_tools!r} permission_mode={job.permission_mode.value!r} " - f"session_id={job.session_id!r}" - ) # ── Resolve provider credentials ─────────────────────────────────────── # Credentials come from the Job when the orchestrator supplies per-workspace @@ -61,15 +56,29 @@ async def run( # (ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL, captured into Settings at startup). # This env fallback is what makes the OSS single-user build work with just an # API key in the environment. An empty base URL means "use the provider default". + # With no key at all, the job runs on the runtime's local-fallback runtime + # (the compact loop over the compose `local` profile's Ollama sidecar) — the + # Job's hosted model is replaced by the locally served one. key_var, base_var = runtime.credential_env_names() effective_api_key = job.api_key or settings.engine_api_key effective_base_url = job.base_url or settings.engine_base_url + if not effective_api_key and settings.local_base_url: + runtime = runtime.local_runtime() + key_var, base_var = runtime.credential_env_names() + model = settings.local_model or model + effective_api_key = "local" # must be non-empty; the local server ignores auth + effective_base_url = settings.local_base_url + log( + f"No API key configured — falling back to local model {model!r} " + f"at {settings.local_base_url!r} via the {runtime.name()!r} runtime" + ) if not effective_api_key: emit_fn( ErrorEvent( message=( f"No API credentials available. Set {key_var} in the engine " - "environment or supply an apiKey on the Job." + "environment, supply an apiKey on the Job, or enable the " + "local model sidecar (COMPOSE_PROFILES=local)." ) ) ) @@ -78,6 +87,12 @@ async def run( if effective_base_url: provider_env[base_var] = effective_base_url + log( + f"Starting. runtime={runtime.name()!r} role={job.role.value!r} model={model!r} " + f"allowed_tools={job.allowed_tools!r} permission_mode={job.permission_mode.value!r} " + f"session_id={job.session_id!r}" + ) + # ── System prompt: base + skills ─────────────────────────────────────── skills_addendum = load_skills(job.skills_path) system_prompt = job.system_prompt + skills_addendum diff --git a/engine/engine_core/runtime.py b/engine/engine_core/runtime.py index bcc118e..eb09ec1 100644 --- a/engine/engine_core/runtime.py +++ b/engine/engine_core/runtime.py @@ -139,6 +139,16 @@ def credential_env_names(self) -> tuple[str, str]: ``ctx.provider_env`` from the resolved credentials.""" return ("ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL") + def local_runtime(self) -> AgentRuntime: + """The runtime used for the local-model fallback (no API key; jobs run + against ``settings.local_base_url``). Defaults to the built-in compact + Ollama loop — heavyweight CLI-based runtimes are impractical against a + small local model (their scaffolding alone is tens of thousands of + prompt tokens).""" + from .local_runtime import LocalOllamaRuntime + + return LocalOllamaRuntime() + # ── Capability flags (let the shared core degrade gracefully) ────────── def supports_thinking(self) -> bool: return False diff --git a/engine/pyproject.toml b/engine/pyproject.toml index 40b0598..40e96f0 100644 --- a/engine/pyproject.toml +++ b/engine/pyproject.toml @@ -16,6 +16,8 @@ dependencies = [ "pydantic-settings>=2.2", # MCP client (deferred-server tool-search router). "mcp>=1.0", + # Local-model fallback runtime (Ollama /api/chat streaming). + "httpx>=0.27", # Claude Agent SDK (bundles the Node Claude Code CLI). "claude-agent-sdk>=0.2", # Observability — OpenTelemetry traces/metrics/logs over OTLP/HTTP. diff --git a/engine/tests/claude/test_runtime.py b/engine/tests/claude/test_runtime.py index 0a59764..33b264f 100644 --- a/engine/tests/claude/test_runtime.py +++ b/engine/tests/claude/test_runtime.py @@ -83,6 +83,39 @@ async def test_missing_key_everywhere_emits_actionable_error(monkeypatch, tmp_pa assert any("ANTHROPIC_API_KEY" in getattr(ev, "message", "") for ev in events) +@pytest.mark.asyncio +async def test_local_fallback_bypasses_sdk(monkeypatch, tmp_path): + # No key anywhere but a local sidecar configured → the run is dispatched to + # the compact local runtime instead of the Claude CLI (whose scaffolding is + # far too heavy for a small local model). The SDK boundary must not be hit. + from engine_core.local_runtime import LocalOllamaRuntime + + assert isinstance(ClaudeRuntime().local_runtime(), LocalOllamaRuntime) + + captured = _patch_sdk(monkeypatch) + + class CapturingLocal(LocalOllamaRuntime): + def __init__(self): + self.ctx = None + + async def run(self, ctx): + self.ctx = ctx + return "sess-local" + + local = CapturingLocal() + monkeypatch.setattr(ClaudeRuntime, "local_runtime", lambda self: local) + settings = _settings(key="") + settings.local_base_url = "http://ollama:11434" + settings.local_model = "qwen2.5:1.5b" + result = await core_run( + _job(tmp_path, apiKey="", model="claude-sonnet-5"), + ClaudeRuntime(), lambda e: None, settings, + ) + assert result == "sess-local" + assert captured == {} # ClaudeAgentOptions never constructed + assert local.ctx.model == "qwen2.5:1.5b" + + @pytest.mark.asyncio async def test_blank_base_url_is_omitted(monkeypatch, tmp_path): captured = _patch_sdk(monkeypatch) diff --git a/engine/tests/core/test_local_runtime.py b/engine/tests/core/test_local_runtime.py new file mode 100644 index 0000000..6d07b5f --- /dev/null +++ b/engine/tests/core/test_local_runtime.py @@ -0,0 +1,274 @@ +"""Tests for the local Ollama runtime — the compact no-API-key agent loop. + +The Ollama HTTP boundary is stubbed (httpx.AsyncClient.stream) so the loop's +streaming, tool dispatch, session persistence, and event emission are exercised +without a server. +""" +import json + +import pytest + +from engine_core import local_runtime as lr_mod +from engine_core.models.events import ( + ResultEvent, + TextDeltaEvent, + ToolResultEvent, + ToolStartEvent, +) +from engine_core.models.job import Job +from engine_core.models.settings import BaseEngineSettings +from engine_core.runner import run as core_run +from engine_core.runtime import AgentRuntime + + +class LocalSettings(BaseEngineSettings): + """Settings with the local fallback configured.""" + + @property + def engine_api_key(self) -> str: + return "" + + +class HostRuntime(AgentRuntime): + """Minimal host runtime whose fallback is the real LocalOllamaRuntime.""" + + def name(self) -> str: + return "host" + + async def run(self, ctx): # pragma: no cover - never used in these tests + raise AssertionError("host runtime must not run") + + +def _stub_stream(monkeypatch, turns: list[list[dict]], captured: list[dict]): + """Stub httpx.AsyncClient.stream to replay one chunk-list per model call.""" + + calls = {"n": 0} + + class FakeResponse: + def __init__(self, chunks): + self._chunks = chunks + + def raise_for_status(self): + pass + + async def aiter_lines(self): + for c in self._chunks: + yield json.dumps(c) + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + class FakeClient: + def __init__(self, timeout=None): + pass + + def stream(self, method, url, json=None): + captured.append({"url": url, "payload": json}) + chunks = turns[min(calls["n"], len(turns) - 1)] + calls["n"] += 1 + return FakeResponse(chunks) + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + monkeypatch.setattr(lr_mod.httpx, "AsyncClient", FakeClient) + + +def _job(tmp_path, **overrides) -> Job: + return Job.model_validate({ + "messages": [{"role": "user", "content": "where is auth handled?"}], + "role": "ask", + "systemPrompt": "You answer questions about the codebase.", + "workspacePath": str(tmp_path), + "permissionMode": "dontAsk", + "allowedTools": ["Read", "Grep"], + **overrides, + }) + + +def _settings(tmp_path) -> LocalSettings: + return LocalSettings( + local_base_url="http://ollama:11434", + local_model="qwen2.5:1.5b", + session_store_path=str(tmp_path / "store"), + ) + + +@pytest.mark.asyncio +async def test_text_only_turn_streams_and_persists(monkeypatch, tmp_path): + captured: list[dict] = [] + _stub_stream(monkeypatch, [[ + {"message": {"role": "assistant", "content": "In "}}, + {"message": {"role": "assistant", "content": "auth.go"}}, + {"done": True, "prompt_eval_count": 100, "eval_count": 5}, + ]], captured) + + events = [] + settings = _settings(tmp_path) + sid = await core_run(_job(tmp_path), HostRuntime(), events.append, settings) + + assert sid is not None + deltas = [e.text for e in events if isinstance(e, TextDeltaEvent)] + assert "".join(deltas) == "In auth.go" + result = next(e for e in events if isinstance(e, ResultEvent)) + assert result.model == "qwen2.5:1.5b" + assert result.usage.input_tokens == 100 and result.usage.output_tokens == 5 + + # The request went to the local server with the compact prompt (the Job's + # system prompt — no skills addendum) and the workspace tools. + payload = captured[0]["payload"] + assert captured[0]["url"] == "http://ollama:11434/api/chat" + assert payload["model"] == "qwen2.5:1.5b" + assert payload["messages"][0] == { + "role": "system", "content": "You answer questions about the codebase.", + } + assert {t["function"]["name"] for t in payload["tools"]} == {"Read", "Grep"} + + # History persisted for resume: next turn with the sessionId replays it. + store_file = tmp_path / "store" / "local-sessions" / f"{sid}.json" + history = json.loads(store_file.read_text()) + assert history[-1] == {"role": "assistant", "content": "In auth.go"} + + +@pytest.mark.asyncio +async def test_tool_call_loop_dispatches_builtin(monkeypatch, tmp_path): + (tmp_path / "auth.go").write_text("package auth\n") + captured: list[dict] = [] + _stub_stream(monkeypatch, [ + [ # turn 1: the model calls Read + {"message": {"role": "assistant", "content": "", "tool_calls": [ + {"function": {"name": "Read", "arguments": {"path": str(tmp_path / "auth.go")}}}, + ]}}, + {"done": True, "prompt_eval_count": 10, "eval_count": 3}, + ], + [ # turn 2: final answer + {"message": {"role": "assistant", "content": "It is package auth."}}, + {"done": True, "prompt_eval_count": 4, "eval_count": 6}, + ], + ], captured) + + events = [] + settings = _settings(tmp_path) + sid = await core_run(_job(tmp_path), HostRuntime(), events.append, settings) + + assert sid is not None + starts = [e for e in events if isinstance(e, ToolStartEvent)] + assert [s.name for s in starts] == ["Read"] + tool_result = next(e for e in events if isinstance(e, ToolResultEvent)) + assert "package auth" in tool_result.result + + # Second model call carries the tool exchange back. + msgs = captured[1]["payload"]["messages"] + assert msgs[-1]["role"] == "tool" and msgs[-1]["tool_name"] == "Read" + assert msgs[-2]["role"] == "assistant" and msgs[-2]["tool_calls"] + + result = next(e for e in events if isinstance(e, ResultEvent)) + assert result.num_turns == 2 + assert result.usage.input_tokens == 14 and result.usage.output_tokens == 9 + + +@pytest.mark.asyncio +async def test_unknown_tool_returns_error_result(monkeypatch, tmp_path): + captured: list[dict] = [] + _stub_stream(monkeypatch, [ + [ + {"message": {"role": "assistant", "content": "", "tool_calls": [ + {"function": {"name": "bogus", "arguments": {}}}, + ]}}, + {"done": True}, + ], + [ + {"message": {"role": "assistant", "content": "done"}}, + {"done": True}, + ], + ], captured) + + events = [] + await core_run(_job(tmp_path), HostRuntime(), events.append, _settings(tmp_path)) + tool_result = next(e for e in events if isinstance(e, ToolResultEvent)) + assert "unknown tool" in tool_result.result + + +@pytest.mark.asyncio +async def test_session_resume_replays_history(monkeypatch, tmp_path): + captured: list[dict] = [] + _stub_stream(monkeypatch, [[ + {"message": {"role": "assistant", "content": "Still auth.go."}}, + {"done": True}, + ]], captured) + + settings = _settings(tmp_path) + store_dir = tmp_path / "store" / "local-sessions" + store_dir.mkdir(parents=True) + (store_dir / "sess-1.json").write_text(json.dumps([ + {"role": "user", "content": "where is auth handled?"}, + {"role": "assistant", "content": "In auth.go"}, + ])) + + events = [] + job = _job(tmp_path, sessionId="sess-1", + messages=[{"role": "user", "content": "and sessions?"}]) + sid = await core_run(job, HostRuntime(), events.append, settings) + assert sid == "sess-1" + + msgs = captured[0]["payload"]["messages"] + # system + 2 history turns + the new user message. + assert [m["role"] for m in msgs] == ["system", "user", "assistant", "user"] + assert msgs[-1]["content"] == "and sessions?" + + history = json.loads((store_dir / "sess-1.json").read_text()) + assert len(history) == 4 and history[-1]["content"] == "Still auth.go." + + +@pytest.mark.asyncio +async def test_string_tool_arguments_are_parsed(monkeypatch, tmp_path): + (tmp_path / "a.txt").write_text("hi\n") + captured: list[dict] = [] + _stub_stream(monkeypatch, [ + [ + {"message": {"role": "assistant", "content": "", "tool_calls": [ + {"function": {"name": "Read", + "arguments": json.dumps({"path": str(tmp_path / "a.txt")})}}, + ]}}, + {"done": True}, + ], + [ + {"message": {"role": "assistant", "content": "ok"}}, + {"done": True}, + ], + ], captured) + + events = [] + await core_run(_job(tmp_path), HostRuntime(), events.append, _settings(tmp_path)) + tool_result = next(e for e in events if isinstance(e, ToolResultEvent)) + assert "hi" in tool_result.result + + +@pytest.mark.asyncio +async def test_oversized_tool_result_is_truncated(monkeypatch, tmp_path): + big = tmp_path / "big.txt" + big.write_text("x" * 50_000) + _stub_stream(monkeypatch, [ + [ + {"message": {"role": "assistant", "content": "", "tool_calls": [ + {"function": {"name": "Read", "arguments": {"path": str(big)}}}, + ]}}, + {"done": True}, + ], + [ + {"message": {"role": "assistant", "content": "ok"}}, + {"done": True}, + ], + ], []) + + events = [] + await core_run(_job(tmp_path), HostRuntime(), events.append, _settings(tmp_path)) + tool_result = next(e for e in events if isinstance(e, ToolResultEvent)) + assert len(tool_result.result) <= lr_mod._MAX_TOOL_RESULT_CHARS + 100 + assert tool_result.result.endswith("[... truncated]") diff --git a/engine/tests/core/test_runner.py b/engine/tests/core/test_runner.py index fde45d1..9a847a6 100644 --- a/engine/tests/core/test_runner.py +++ b/engine/tests/core/test_runner.py @@ -177,3 +177,137 @@ async def test_attachment_force_enables_read(tmp_path): assert "Read" in rt.ctx.requested_tools assert rt.ctx.permission_mode == "dontAsk" assert [d.name for d in rt.ctx.builtin_tools.workspace] == ["Read", "Glob", "Grep"] + + +# --- Local model fallback (no API key → swap to the local runtime) --- + + +class FakeLocalRuntime(FakeRuntime): + """Capturing stand-in for the local-fallback runtime.""" + + def name(self) -> str: + return "fake-local" + + def credential_env_names(self) -> tuple[str, str]: + return ("LOCAL_API_KEY", "LOCAL_BASE_URL") + + async def run(self, ctx: RunContext) -> str | None: + self.ctx = ctx + return "sess-local" + + +class SwappingRuntime(FakeRuntime): + """FakeRuntime whose local fallback is a capturing FakeLocalRuntime.""" + + def __init__(self): + super().__init__() + self.local = FakeLocalRuntime() + + def local_runtime(self) -> FakeLocalRuntime: + return self.local + + +@pytest.mark.asyncio +async def test_local_fallback_swaps_runtime(tmp_path): + # No Job key, no env key, but a local server is configured → the run is + # dispatched to the runtime's local_runtime() with the local model. + rt = SwappingRuntime() + events, emit = _collect() + job = Job.model_validate({ + "messages": [{"role": "user", "content": "hi"}], + "workspacePath": str(tmp_path), + "model": "claude-sonnet-5", + }) + settings = FakeSettings(local_base_url="http://ollama:11434", local_model="qwen2.5:1.5b") + sid = await run(job, rt, emit, settings) + assert sid == "sess-local" + assert not any(isinstance(e, ErrorEvent) for e in events) + assert rt.ctx is None # the original runtime never ran + assert rt.local.ctx.model == "qwen2.5:1.5b" + assert rt.local.ctx.provider_env == { + "LOCAL_API_KEY": "local", + "LOCAL_BASE_URL": "http://ollama:11434", + } + + +@pytest.mark.asyncio +async def test_local_fallback_without_local_model_keeps_job_model(tmp_path): + rt = SwappingRuntime() + _, emit = _collect() + job = Job.model_validate({ + "messages": [{"role": "user", "content": "hi"}], + "workspacePath": str(tmp_path), + "model": "some-model", + }) + await run(job, rt, emit, FakeSettings(local_base_url="http://ollama:11434")) + assert rt.local.ctx.model == "some-model" + + +@pytest.mark.asyncio +async def test_local_runtime_drops_interactive_tools(tmp_path): + # The default local runtime doesn't support interactive tools, so AskChoice + # must not be registered even when the Job allows it. + class NonInteractiveLocal(FakeLocalRuntime): + def supports_interactive(self) -> bool: + return False + + rt = SwappingRuntime() + rt.local = NonInteractiveLocal() + _, emit = _collect() + job = Job.model_validate({ + "messages": [{"role": "user", "content": "hi"}], + "workspacePath": str(tmp_path), + "allowedTools": ["Read", "AskChoice"], + "permissionMode": "dontAsk", + }) + await run(job, rt, emit, FakeSettings(local_base_url="http://ollama:11434")) + assert rt.local.ctx.builtin_tools.interactive == [] + + +@pytest.mark.asyncio +async def test_job_key_wins_over_local_fallback(tmp_path): + rt = SwappingRuntime() + _, emit = _collect() + job = Job.model_validate({ + "messages": [{"role": "user", "content": "hi"}], + "workspacePath": str(tmp_path), + "model": "claude-sonnet-5", + "apiKey": "job-key", + }) + settings = FakeSettings(local_base_url="http://ollama:11434", local_model="qwen2.5:1.5b") + await run(job, rt, emit, settings) + assert rt.local.ctx is None # local runtime not used + assert rt.ctx.model == "claude-sonnet-5" + assert rt.ctx.provider_env == {"FAKE_API_KEY": "job-key"} + + +@pytest.mark.asyncio +async def test_env_key_wins_over_local_fallback(tmp_path): + rt = SwappingRuntime() + _, emit = _collect() + job = Job.model_validate({ + "messages": [{"role": "user", "content": "hi"}], + "workspacePath": str(tmp_path), + "model": "claude-sonnet-5", + }) + settings = FakeSettings( + fake_key="env-key", + local_base_url="http://ollama:11434", local_model="qwen2.5:1.5b", + ) + await run(job, rt, emit, settings) + assert rt.local.ctx is None + assert rt.ctx.model == "claude-sonnet-5" + assert rt.ctx.provider_env == {"FAKE_API_KEY": "env-key"} + + +@pytest.mark.asyncio +async def test_no_key_and_no_local_still_errors(tmp_path): + rt = FakeRuntime() + events, emit = _collect() + job = Job.model_validate({ + "messages": [{"role": "user", "content": "hi"}], + "workspacePath": str(tmp_path), + }) + sid = await run(job, rt, emit, FakeSettings()) + assert sid is None + assert any(isinstance(e, ErrorEvent) and "COMPOSE_PROFILES=local" in e.message for e in events) diff --git a/indexer/.env.example b/indexer/.env.example index c747f01..36b2daf 100644 --- a/indexer/.env.example +++ b/indexer/.env.example @@ -14,12 +14,19 @@ DEXIASK_REPOS_CONFIG_PATH=repos.yaml # --- Stores --- DEXIASK_QDRANT_URL=http://localhost:25055 -# --- Embeddings (hosted, code-specialized). Same model/dim across all repos. --- -DEXIASK_EMBEDDING_PROVIDER=voyage +# --- Embeddings. Same model/dim across all repos. --- +# "auto" picks the first configured provider: voyage (key) → openai (key) → +# ollama (base URL — the local no-key sidecar). Or pin one explicitly: +# voyage | openai | ollama | hash. +DEXIASK_EMBEDDING_PROVIDER=auto DEXIASK_EMBEDDING_MODEL=voyage-code-3 DEXIASK_EMBEDDING_DIM=1024 DEXIASK_VOYAGE_API_KEY= # DEXIASK_OPENAI_API_KEY= +# Local Ollama sidecar (no API key). The embedding model must produce +# DEXIASK_EMBEDDING_DIM-sized vectors. +# DEXIASK_OLLAMA_BASE_URL=http://ollama:11434 +# DEXIASK_OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b # --- Remote repo auth (for private HTTPS clones; optional) --- # DEXIASK_GIT_TOKEN= diff --git a/indexer/indexer/embedding/factory.py b/indexer/indexer/embedding/factory.py index 927d554..acb7069 100644 --- a/indexer/indexer/embedding/factory.py +++ b/indexer/indexer/embedding/factory.py @@ -1,12 +1,19 @@ """Build an embedding provider from settings.""" from __future__ import annotations +import logging + from ..settings import Settings from .base import EmbeddingProvider +log = logging.getLogger("indexer.embedding") + def build_provider(settings: Settings) -> EmbeddingProvider: provider = settings.embedding_provider.lower() + if provider == "auto": + provider = _resolve_auto(settings) + log.info("Embedding provider 'auto' resolved to %r", provider) if provider == "voyage": from .voyage import VoyageProvider @@ -25,8 +32,32 @@ def build_provider(settings: Settings) -> EmbeddingProvider: dim=settings.embedding_dim, batch_size=settings.embedding_batch_size, ) + if provider == "ollama": + from .ollama import OllamaProvider + + return OllamaProvider( + base_url=settings.ollama_base_url, + model=settings.ollama_embedding_model, + dim=settings.embedding_dim, + batch_size=settings.embedding_batch_size, + ) if provider == "hash": from .hashprovider import HashEmbeddingProvider return HashEmbeddingProvider(model=settings.embedding_model, dim=settings.embedding_dim) raise ValueError(f"unknown embedding provider {settings.embedding_provider!r}") + + +def _resolve_auto(settings: Settings) -> str: + """First configured provider wins: hosted keys beat the local sidecar.""" + if settings.voyage_api_key: + return "voyage" + if settings.openai_api_key: + return "openai" + if settings.ollama_base_url: + return "ollama" + raise ValueError( + "no embedding provider configured: set DEXIASK_VOYAGE_API_KEY / " + "DEXIASK_OPENAI_API_KEY, or enable the local Ollama sidecar " + "(COMPOSE_PROFILES=local)" + ) diff --git a/indexer/indexer/embedding/ollama.py b/indexer/indexer/embedding/ollama.py new file mode 100644 index 0000000..3b0e04c --- /dev/null +++ b/indexer/indexer/embedding/ollama.py @@ -0,0 +1,103 @@ +"""Ollama embedding provider (local, no API key; default: qwen3-embedding:0.6b). + +Backs the no-API-key local mode: the docker-compose ``local`` profile runs an +Ollama sidecar with the embedding model baked into the image, and the ``auto`` +provider falls back here when no hosted-provider key is configured. +""" +from __future__ import annotations + +import logging +import time + +import httpx + +from ._batching import prepare_batches +from .base import EmbeddingProvider + +log = logging.getLogger("indexer.embedding") + +# Ollama has no per-request input caps like the hosted APIs, but batches are +# processed synchronously — keep them modest so one request can't stall an +# index pass, and truncate inputs well under the model's context window. +_MAX_BATCH_TOKENS = 50_000 +_MAX_INPUT_CHARS = 30_000 # ~7.5K tokens, safely under qwen3-embedding's 32K context + + +class OllamaProvider(EmbeddingProvider): + """Embeds via a local Ollama server's native ``/api/embed`` endpoint. + + The constructor never touches the network (the sidecar may still be + starting when the indexer boots); each batch retries with exponential + backoff, which also absorbs that startup race. The returned vector size is + validated against the pinned collection dimension so a misconfigured model + fails loudly instead of writing vectors of the wrong shape. + """ + + def __init__( + self, + base_url: str, + model: str = "qwen3-embedding:0.6b", + dim: int = 1024, + max_retries: int = 4, + sleep=time.sleep, + batch_size: int = 128, + max_batch_tokens: int = _MAX_BATCH_TOKENS, + max_input_chars: int = _MAX_INPUT_CHARS, + timeout_s: float = 120.0, + ) -> None: + if not base_url: + raise ValueError("Ollama base URL is required (DEXIASK_OLLAMA_BASE_URL)") + self.model = model + self.dim = dim + self._base_url = base_url.rstrip("/") + self._max_retries = max_retries + self._sleep = sleep + self._batch_size = max(1, batch_size) + self._max_batch_tokens = max_batch_tokens + self._max_input_chars = max_input_chars + self._timeout_s = timeout_s + + def _embed(self, texts: list[str]) -> list[list[float]]: + last: Exception | None = None + for attempt in range(self._max_retries): + try: + resp = httpx.post( + f"{self._base_url}/api/embed", + json={"model": self.model, "input": texts, "truncate": True}, + timeout=self._timeout_s, + ) + resp.raise_for_status() + embeddings = resp.json()["embeddings"] + if embeddings and len(embeddings[0]) != self.dim: + raise ValueError( + f"Ollama model {self.model!r} returned {len(embeddings[0])}-dim " + f"vectors but the collection is pinned to {self.dim} " + "(DEXIASK_EMBEDDING_DIM); pick a matching model or dim" + ) + return embeddings + except ValueError: + raise # dim mismatch is a config error, not transient + except Exception as e: # noqa: BLE001 - retry any transient provider error + last = e + if attempt < self._max_retries - 1: + delay = 4 * (2**attempt) + log.warning("Ollama embed failed (attempt %d), retrying in %ds: %s", + attempt + 1, delay, e) + self._sleep(delay) + raise last # type: ignore[misc] + + def embed_documents(self, texts: list[str]) -> list[list[float]]: + if not texts: + return [] + out: list[list[float]] = [] + for batch in prepare_batches( + texts, + max_items=self._batch_size, + max_tokens=self._max_batch_tokens, + max_chars=self._max_input_chars, + ): + out.extend(self._embed(batch)) + return out + + def embed_query(self, text: str) -> list[float]: + return self._embed([text[: self._max_input_chars]])[0] diff --git a/indexer/indexer/settings.py b/indexer/indexer/settings.py index e11d498..3be3c53 100644 --- a/indexer/indexer/settings.py +++ b/indexer/indexer/settings.py @@ -35,15 +35,22 @@ class Settings(BaseSettings): qdrant_url: str = "http://localhost:6333" # --- Embeddings --- - # One of: "voyage", "openai", "hash". Must be identical across repos so their - # vectors live in a comparable space for cross-repo search. - embedding_provider: str = "voyage" + # One of: "auto", "voyage", "openai", "ollama", "hash". Must be identical + # across repos so their vectors live in a comparable space for cross-repo + # search. "auto" picks the first configured provider: voyage (if key) → + # openai (if key) → ollama (if base URL, i.e. the local sidecar is up). + embedding_provider: str = "auto" embedding_model: str = "voyage-code-3" # Vector dimension; pinned and stored in each collection's metadata. embedding_dim: int = 1024 embedding_batch_size: int = 128 voyage_api_key: str = "" openai_api_key: str = "" + # Local Ollama sidecar (no API key). Set by the compose `local` profile; + # empty = unavailable. The model has its own setting because + # ``embedding_model`` stays pinned to the hosted default. + ollama_base_url: str = "" + ollama_embedding_model: str = "qwen3-embedding:0.6b" # --- Remote repo auth --- # Default token used to clone/fetch private HTTPS repos when a repo declares a diff --git a/indexer/tests/test_embedding.py b/indexer/tests/test_embedding.py index 63828de..1264971 100644 --- a/indexer/tests/test_embedding.py +++ b/indexer/tests/test_embedding.py @@ -250,3 +250,173 @@ def embed(self, *a, **k): p = VoyageProvider(api_key="k", dim=8, max_retries=2, sleep=lambda s: None) with pytest.raises(RuntimeError): p.embed_query("x") + + +# --- Ollama (local sidecar, no API key) --- + + +class _FakeResponse: + def __init__(self, embeddings): + self._embeddings = embeddings + + def raise_for_status(self): + pass + + def json(self): + return {"embeddings": self._embeddings} + + +def _stub_ollama_post(monkeypatch, handler): + from indexer.embedding import ollama as ollama_mod + monkeypatch.setattr(ollama_mod.httpx, "post", handler) + + +def test_ollama_requires_base_url(): + from indexer.embedding.ollama import OllamaProvider + with pytest.raises(ValueError): + OllamaProvider(base_url="") + + +def test_ollama_constructor_is_offline(monkeypatch): + # Constructing the provider must never touch the network — the sidecar may + # still be starting when the indexer boots. + def explode(*a, **k): + raise AssertionError("network touched in constructor") + + _stub_ollama_post(monkeypatch, explode) + from indexer.embedding.ollama import OllamaProvider + OllamaProvider(base_url="http://ollama:11434") + + +def test_ollama_embeds_documents_and_query(monkeypatch): + captured = {} + + def post(url, json, timeout): + captured["url"] = url + captured["json"] = json + return _FakeResponse([[0.0] * 8 for _ in json["input"]]) + + _stub_ollama_post(monkeypatch, post) + from indexer.embedding.ollama import OllamaProvider + p = OllamaProvider(base_url="http://ollama:11434/", model="qwen3-embedding:0.6b", dim=8) + assert p.embed_documents([]) == [] + docs = p.embed_documents(["a", "b"]) + assert len(docs) == 2 and len(docs[0]) == 8 + assert captured["url"] == "http://ollama:11434/api/embed" + assert captured["json"]["model"] == "qwen3-embedding:0.6b" + assert captured["json"]["truncate"] is True + assert len(p.embed_query("find auth")) == 8 + + +def test_ollama_retries_then_succeeds(monkeypatch): + state = {"calls": 0} + + def post(url, json, timeout): + state["calls"] += 1 + if state["calls"] == 1: + raise RuntimeError("connection refused") + return _FakeResponse([[0.0] * 8 for _ in json["input"]]) + + _stub_ollama_post(monkeypatch, post) + from indexer.embedding.ollama import OllamaProvider + p = OllamaProvider(base_url="http://ollama:11434", dim=8, max_retries=3, sleep=lambda s: None) + assert len(p.embed_query("x")) == 8 + assert state["calls"] == 2 + + +def test_ollama_raises_after_exhausting_retries(monkeypatch): + def post(url, json, timeout): + raise RuntimeError("still down") + + _stub_ollama_post(monkeypatch, post) + from indexer.embedding.ollama import OllamaProvider + p = OllamaProvider(base_url="http://ollama:11434", dim=8, max_retries=2, sleep=lambda s: None) + with pytest.raises(RuntimeError): + p.embed_query("x") + + +def test_ollama_dim_mismatch_is_not_retried(monkeypatch): + state = {"calls": 0} + + def post(url, json, timeout): + state["calls"] += 1 + return _FakeResponse([[0.0] * 768 for _ in json["input"]]) + + _stub_ollama_post(monkeypatch, post) + from indexer.embedding.ollama import OllamaProvider + p = OllamaProvider(base_url="http://ollama:11434", dim=1024, sleep=lambda s: None) + with pytest.raises(ValueError, match="768"): + p.embed_query("x") + assert state["calls"] == 1 # config error, not transient — no retries + + +def test_ollama_batches_documents(monkeypatch): + calls = [] + + def post(url, json, timeout): + calls.append(json["input"]) + return _FakeResponse([[0.0] * 4 for _ in json["input"]]) + + _stub_ollama_post(monkeypatch, post) + from indexer.embedding.ollama import OllamaProvider + p = OllamaProvider(base_url="http://ollama:11434", dim=4, batch_size=2) + out = p.embed_documents([f"c{i}" for i in range(5)]) + assert len(out) == 5 + assert [len(c) for c in calls] == [2, 2, 1] + + +def test_factory_builds_ollama(): + prov = build_provider(Settings( + embedding_provider="ollama", + ollama_base_url="http://ollama:11434", + ollama_embedding_model="qwen3-embedding:0.6b", + embedding_dim=8, + )) + assert prov.dim == 8 + assert prov.model == "qwen3-embedding:0.6b" + + +# --- "auto" provider resolution: hosted keys beat the local sidecar --- + + +def test_factory_auto_prefers_voyage_key(monkeypatch): + _stub_voyage(monkeypatch, {}) + prov = build_provider(Settings( + embedding_provider="auto", voyage_api_key="k", + ollama_base_url="http://ollama:11434", embedding_dim=8, + )) + from indexer.embedding.voyage import VoyageProvider + assert isinstance(prov, VoyageProvider) + + +def test_factory_auto_falls_back_to_openai_key(monkeypatch): + mod = types.ModuleType("openai") + + class OpenAI: + def __init__(self, api_key): + pass + + mod.OpenAI = OpenAI + monkeypatch.setitem(sys.modules, "openai", mod) + prov = build_provider(Settings( + embedding_provider="auto", openai_api_key="k", + ollama_base_url="http://ollama:11434", embedding_dim=8, + )) + from indexer.embedding.openai import OpenAIProvider + assert isinstance(prov, OpenAIProvider) + + +def test_factory_auto_falls_back_to_ollama(): + prov = build_provider(Settings( + embedding_provider="auto", ollama_base_url="http://ollama:11434", embedding_dim=8, + )) + from indexer.embedding.ollama import OllamaProvider + assert isinstance(prov, OllamaProvider) + + +def test_factory_auto_with_nothing_configured_raises(): + with pytest.raises(ValueError, match="no embedding provider configured"): + build_provider(Settings( + embedding_provider="auto", + voyage_api_key="", openai_api_key="", ollama_base_url="", + )) diff --git a/ollama/Dockerfile b/ollama/Dockerfile new file mode 100644 index 0000000..1cf6aa5 --- /dev/null +++ b/ollama/Dockerfile @@ -0,0 +1,54 @@ +# Ollama sidecar for local (no-API-key) mode. +# +# Models are pulled at BUILD time and live in plain image layers, so nothing is +# downloaded after the stack is up. The engine's local runtime drives the +# native /api/chat with tool calling; pin the base version for reproducibility. +ARG OLLAMA_VERSION=0.31.2 +FROM ollama/ollama:${OLLAMA_VERSION} + +# A small NON-reasoning model with tool support: reasoning models (qwen3) burn +# the whole turn "thinking" on CPU, and the Anthropic-compat API can't turn +# that off. +ARG TEXT_MODEL=qwen2.5:1.5b +ARG EMBED_MODEL=qwen3-embedding:0.6b +# Tuning baked into the TEXT model only (via a derived Modelfile) — not global +# env — so the tiny embedding model keeps its small default context instead of +# reserving a huge KV cache. +# - num_ctx: Ollama's 4096 default would truncate the agent's system prompt; +# 16K fits it while keeping the KV cache modest on small machines. +# - num_thread: llama.cpp defaults to every core, but on laptops (and Docker +# VMs exposing efficiency cores) thread contention makes generation 10-300x +# SLOWER than a few pinned threads. 4 is a safe fast default everywhere; +# raise it on real many-core servers. +ARG TEXT_CONTEXT=16384 +ARG TEXT_THREADS=4 + +# The base image declares VOLUME /root/.ollama — anything pulled there during +# build is shadowed by an anonymous volume at runtime. Store models in /models +# instead so the build-time pulls survive into the running container. +ENV OLLAMA_MODELS=/models + +# Pull each model in its own layer (embed ~640 MB, text ~1 GB) so a text-model +# change doesn't invalidate the embed layer in the build cache. `ollama pull` +# needs a running server; start one for the duration of the RUN step. +RUN ollama serve & \ + until ollama list >/dev/null 2>&1; do sleep 1; done; \ + ollama pull "${EMBED_MODEL}" +# Pull the text model, then re-tag it with the tuning baked in so the Anthropic +# API (which sends neither num_ctx nor num_thread) gets them without global env. +RUN ollama serve & \ + until ollama list >/dev/null 2>&1; do sleep 1; done; \ + ollama pull "${TEXT_MODEL}"; \ + printf 'FROM %s\nPARAMETER num_ctx %s\nPARAMETER num_thread %s\n' \ + "${TEXT_MODEL}" "${TEXT_CONTEXT}" "${TEXT_THREADS}" > /tmp/Modelfile; \ + ollama create "${TEXT_MODEL}" -f /tmp/Modelfile; \ + rm -f /tmp/Modelfile + +# Pre-load the text model into RAM on startup (background, keep-alive forever) so +# the first chat turn doesn't pay the cold model-load latency inside the engine's +# no-output watchdog. The entrypoint keeps Ollama's own startup behaviour. +ENV TEXT_MODEL=${TEXT_MODEL} +COPY entrypoint.sh /usr/local/bin/dexiask-entrypoint.sh +RUN chmod +x /usr/local/bin/dexiask-entrypoint.sh +ENTRYPOINT ["/usr/local/bin/dexiask-entrypoint.sh"] +CMD ["serve"] diff --git a/ollama/entrypoint.sh b/ollama/entrypoint.sh new file mode 100644 index 0000000..91da322 --- /dev/null +++ b/ollama/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Start Ollama, then warm the text model into RAM in the background so the first +# chat turn skips the cold model-load (which otherwise exceeds the engine's +# no-output watchdog on slower hosts). Warming is best-effort — the server is +# usable for embeddings and other models regardless. +set -e + +if [ -n "${TEXT_MODEL}" ]; then + ( + until ollama list >/dev/null 2>&1; do sleep 1; done + echo "[dexiask] pre-loading text model ${TEXT_MODEL} into RAM..." + # A one-shot prompt triggers the load; the server keeps it resident per + # OLLAMA_KEEP_ALIVE (=-1 in compose → forever). + ollama run "${TEXT_MODEL}" "hi" >/dev/null 2>&1 \ + && echo "[dexiask] text model ${TEXT_MODEL} resident" \ + || echo "[dexiask] warm-up of ${TEXT_MODEL} failed (non-fatal)" + ) & +fi + +exec /bin/ollama "$@"