diff --git a/apps/api/src/handlers/inference/__tests__/inference-gateway.test.ts b/apps/api/src/handlers/inference/__tests__/inference-gateway.test.ts index 998a33729..085691a5d 100644 --- a/apps/api/src/handlers/inference/__tests__/inference-gateway.test.ts +++ b/apps/api/src/handlers/inference/__tests__/inference-gateway.test.ts @@ -7,10 +7,12 @@ const { mockFindTaskRun, mockResolveModelProviderEnvValue, mockGetFreshChatGptAccessToken, + mockRecordLlmUsage, } = vi.hoisted(() => ({ mockFindTaskRun: vi.fn(), mockResolveModelProviderEnvValue: vi.fn(), mockGetFreshChatGptAccessToken: vi.fn(), + mockRecordLlmUsage: vi.fn(), })); vi.mock('@roomote/db/server', () => ({ @@ -25,6 +27,10 @@ vi.mock('@roomote/db/server', () => ({ getFreshChatGptAccessToken: mockGetFreshChatGptAccessToken, })); +vi.mock('@roomote/sdk/server', () => ({ + recordLlmUsage: mockRecordLlmUsage, +})); + import { inference } from '../index'; function createApp(authContext: Variables['authContext']) { @@ -300,6 +306,118 @@ describe('inference gateway', () => { ); }); + it('proxies a configured LiteLLM endpoint with its optional key', async () => { + const fetchMock = stubUpstreamFetch(); + mockResolveModelProviderEnvValue.mockImplementation( + async (names: string | readonly string[]) => { + const nameList = typeof names === 'string' ? [names] : names; + + if (nameList.includes('LITELLM_BASE_URL')) { + return 'http://litellm.internal:4000/v1/'; + } + + return nameList.includes('LITELLM_API_KEY') ? 'litellm-key' : undefined; + }, + ); + + const response = await postMessages( + createApp(createRunToken()), + '/api/inference/litellm/v1/chat/completions', + ); + + expect(response.status).toBe(200); + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('http://litellm.internal:4000/v1/chat/completions'); + expect(new Headers(init.headers).get('authorization')).toBe( + 'Bearer litellm-key', + ); + }); + + it('records a positive LiteLLM response cost without delaying the proxy', async () => { + stubUpstreamFetch( + new Response('data: [DONE]\n\n', { + headers: { + 'content-type': 'text/event-stream', + 'x-litellm-response-cost': '0.000321', + 'x-litellm-model-group': 'coding', + }, + }), + ); + mockFindTaskRun.mockResolvedValue({ taskId: 'task-1' }); + mockResolveModelProviderEnvValue.mockImplementation( + async (names: string | readonly string[]) => { + const nameList = typeof names === 'string' ? [names] : names; + return nameList.includes('LITELLM_BASE_URL') + ? 'http://litellm.internal:4000' + : 'litellm-key'; + }, + ); + + const response = await postMessages( + createApp(createRunToken()), + '/api/inference/litellm/v1/chat/completions', + ); + + expect(response.status).toBe(200); + await vi.waitFor(() => { + expect(mockRecordLlmUsage).toHaveBeenCalledWith( + expect.objectContaining({ + eventKey: expect.stringMatching(/^inference-gateway:/u), + taskId: 'task-1', + runId: 42, + providerId: 'litellm', + modelId: 'litellm/coding', + costMicroUsd: 321, + costSource: 'litellm_gateway', + }), + ); + }); + }); + + it('proxies a configured Ollama endpoint without an API key', async () => { + const fetchMock = stubUpstreamFetch(); + mockResolveModelProviderEnvValue.mockImplementation( + async (names: string | readonly string[]) => { + const nameList = typeof names === 'string' ? [names] : names; + + return nameList.includes('OLLAMA_BASE_URL') + ? 'http://ollama.internal:11434' + : undefined; + }, + ); + + const response = await postMessages( + createApp(createRunToken()), + '/api/inference/ollama/v1/chat/completions', + ); + + expect(response.status).toBe(200); + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('http://ollama.internal:11434/v1/chat/completions'); + expect(new Headers(init.headers).get('authorization')).toBeNull(); + }); + + it('rejects malformed dynamic upstream URLs before fetching', async () => { + const fetchMock = stubUpstreamFetch(); + mockResolveModelProviderEnvValue.mockImplementation( + async (names: string | readonly string[]) => { + const nameList = typeof names === 'string' ? [names] : names; + + return nameList.includes('VLLM_BASE_URL') + ? 'https://token@example.test?redirect=https://evil.test' + : undefined; + }, + ); + + const response = await postMessages( + createApp(createRunToken()), + '/api/inference/vllm/v1/chat/completions', + ); + + expect(response.status).toBe(500); + expect(fetchMock).not.toHaveBeenCalled(); + }); + it('rejects invalid Bedrock regions before building the upstream URL', async () => { const fetchMock = stubUpstreamFetch(); mockResolveModelProviderEnvValue.mockImplementation( diff --git a/apps/api/src/handlers/inference/index.ts b/apps/api/src/handlers/inference/index.ts index 8148f29ca..bee4fa04f 100644 --- a/apps/api/src/handlers/inference/index.ts +++ b/apps/api/src/handlers/inference/index.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono'; import { formatSingleLineLog } from '@roomote/types'; import { db, eq, taskRuns } from '@roomote/db/server'; +import { recordLlmUsage } from '@roomote/sdk/server'; import type { Variables } from '../../types'; import { fetchWithLongLivedStreamDispatcher } from '../long-lived-fetch'; @@ -49,6 +50,59 @@ const REQUEST_HEADER_DENYLIST = new Set([ 'x-real-ip', ]); +function recordLiteLlmResponseCost(options: { + requestId: string; + runId: number; + headers: Headers; +}): void { + const cost = Number(options.headers.get('x-litellm-response-cost')); + + if (!Number.isFinite(cost) || cost <= 0) { + return; + } + + const modelGroup = options.headers.get('x-litellm-model-group')?.trim(); + + void db.query.taskRuns + .findFirst({ + where: eq(taskRuns.id, options.runId), + columns: { taskId: true }, + }) + .then((run) => { + if (!run?.taskId) { + return undefined; + } + + return recordLlmUsage({ + eventKey: `inference-gateway:${options.requestId}`, + source: 'inference-gateway', + usageType: 'inference', + taskId: run.taskId, + runId: options.runId, + providerId: 'litellm', + modelId: modelGroup ? `litellm/${modelGroup}` : null, + inputTokens: null, + outputTokens: null, + reasoningTokens: null, + cacheReadTokens: null, + cacheWriteTokens: null, + totalTokens: null, + contextTokens: null, + costMicroUsd: Math.round(cost * 1_000_000), + costSource: 'litellm_gateway', + }); + }) + .catch((error) => { + console.warn( + formatSingleLineLog('Failed to record LiteLLM response cost', { + requestId: options.requestId, + runId: options.runId, + error: error instanceof Error ? error.message : String(error), + }), + ); + }); +} + function buildUpstreamRequestHeaders( requestHeaders: Headers, injectedHeaders: Record, @@ -188,6 +242,14 @@ inference.on(['POST', 'GET'], '/:provider/*', async (c) => { }, ); + if (providerId === 'litellm') { + recordLiteLlmResponseCost({ + requestId, + runId: auth.runId, + headers: upstreamResponse.headers, + }); + } + if (!upstreamResponse.ok) { console.warn( formatSingleLineLog(`${logPrefix} Upstream returned non-OK status`, { diff --git a/apps/api/src/handlers/inference/registry.ts b/apps/api/src/handlers/inference/registry.ts index 7e967e785..4f07186ea 100644 --- a/apps/api/src/handlers/inference/registry.ts +++ b/apps/api/src/handlers/inference/registry.ts @@ -46,7 +46,7 @@ export async function resolveGatewayUpstream( resolveProviderUpstreamBaseUrl(provider), ]); - if (!apiKey) { + if (!apiKey && !provider.optionalApiKey) { return { ok: false, status: 404, @@ -58,12 +58,15 @@ export async function resolveGatewayUpstream( ok: true, resolved: { upstreamUrl: `${upstreamBaseUrl}${upstreamPath}${search}`, - headers: { - [provider.authHeader.name]: formatProviderAuthHeaderValue( - provider, - apiKey, - ), - }, + headers: + apiKey && provider.authHeader + ? { + [provider.authHeader.name]: formatProviderAuthHeaderValue( + provider, + apiKey, + ), + } + : {}, }, }; } @@ -81,9 +84,9 @@ async function resolveChatGptUpstream( }; } - const upstreamUrl = `${provider.upstreamBaseUrl}${provider.collapseToPath ?? ''}`; + const upstreamUrl = `${provider.upstreamBaseUrl!}${provider.collapseToPath ?? ''}`; const headers: Record = { - [provider.authHeader.name]: formatProviderAuthHeaderValue( + [provider.authHeader!.name]: formatProviderAuthHeaderValue( provider, token.access, ), @@ -150,8 +153,25 @@ function hasTraversalOrEncodedSlash(upstreamPath: string): boolean { async function resolveProviderUpstreamBaseUrl( provider: InferenceGatewayProvider, ): Promise { + if (provider.upstreamBaseUrlEnvVarName) { + const configuredBaseUrl = await resolveModelProviderEnvValue([ + provider.upstreamBaseUrlEnvVarName, + ]); + + if (!configuredBaseUrl) { + throw new Error( + `${provider.upstreamBaseUrlEnvVarName} must be configured for ${provider.name}.`, + ); + } + + return validateDynamicUpstreamBaseUrl( + configuredBaseUrl, + provider.upstreamBaseUrlEnvVarName, + ); + } + if (!provider.region) { - return provider.upstreamBaseUrl; + return provider.upstreamBaseUrl!; } const region = @@ -164,12 +184,56 @@ async function resolveProviderUpstreamBaseUrl( ); } - return provider.upstreamBaseUrl.replace('{region}', region); + return provider.upstreamBaseUrl!.replace('{region}', region); +} + +function validateDynamicUpstreamBaseUrl( + value: string, + envVarName: string, +): string { + let url: URL; + + try { + url = new URL(value); + } catch { + throw new Error(`${envVarName} must be an absolute HTTP(S) URL.`); + } + + if ( + (url.protocol !== 'http:' && url.protocol !== 'https:') || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error( + `${envVarName} must be an HTTP(S) URL without credentials, query parameters, or fragments.`, + ); + } + + const normalizedPath = stripDynamicEndpointVersionSuffix(url.pathname); + url.pathname = normalizedPath; + + return url.toString().replace(/\/+$/u, ''); +} + +function stripDynamicEndpointVersionSuffix(pathname: string): string { + let end = pathname.length; + + while (end > 0 && pathname.charCodeAt(end - 1) === 47 /* '/' */) { + end -= 1; + } + + const withoutTrailingSlashes = pathname.slice(0, end); + + return withoutTrailingSlashes.endsWith('/v1') + ? withoutTrailingSlashes.slice(0, -3) || '/' + : withoutTrailingSlashes || '/'; } function formatProviderAuthHeaderValue( provider: InferenceGatewayProvider, apiKey: string, ): string { - return provider.authHeader.scheme === 'bearer' ? `Bearer ${apiKey}` : apiKey; + return provider.authHeader?.scheme === 'bearer' ? `Bearer ${apiKey}` : apiKey; } diff --git a/apps/docs/docs.json b/apps/docs/docs.json index 7c8684afe..7f5e57d33 100644 --- a/apps/docs/docs.json +++ b/apps/docs/docs.json @@ -43,7 +43,12 @@ { "group": "Models and Inference", "root": "models", - "pages": [] + "expanded": true, + "pages": [ + "providers/inference/litellm", + "providers/inference/ollama", + "providers/inference/vllm" + ] }, { "group": "Communications", diff --git a/apps/docs/environment-variables.mdx b/apps/docs/environment-variables.mdx index f49697575..ae68eea36 100644 --- a/apps/docs/environment-variables.mdx +++ b/apps/docs/environment-variables.mdx @@ -188,6 +188,11 @@ as per-task auth tokens or workspace paths. | `GOOGLE_GENERATIVE_AI_API_KEY` | Provider key | Alternate Google/Gemini provider key forwarded when configured or inferred. | | `AWS_BEARER_TOKEN_BEDROCK` | Provider key | Amazon Bedrock Mantle API key. Can also be saved from **Settings > Models**. | | `AWS_REGION` | Provider key | AWS region where the Bedrock API key was created. Defaults to `us-east-1` at runtime when unset. | +| `LITELLM_BASE_URL` | LiteLLM | LiteLLM endpoint URL, usually including its `/v1` path. | +| `LITELLM_API_KEY` | LiteLLM | LiteLLM gateway API key. Required when configuring LiteLLM. | +| `OLLAMA_BASE_URL` | Ollama | Ollama endpoint URL. Use the service root, for example `http://ollama:11434`. | +| `VLLM_BASE_URL` | vLLM | vLLM OpenAI-compatible endpoint URL, usually including its `/v1` path. | +| `VLLM_API_KEY` | Optional | Bearer API key for a vLLM endpoint that requires authentication. | ### Sandbox providers diff --git a/apps/docs/models.mdx b/apps/docs/models.mdx index e330c3f8e..d10fec878 100644 --- a/apps/docs/models.mdx +++ b/apps/docs/models.mdx @@ -17,7 +17,8 @@ Configure models from **Settings > Models**. An inference provider is the service that hosts or routes model calls. Roomote supports providers such as OpenRouter, Vercel AI Gateway, Baseten, Together AI, OpenAI, Anthropic, Moonshot AI, MiniMax, OpenCode, Amazon Bedrock, -Google Gemini, xAI, and ChatGPT subscriptions. +Google Gemini, xAI, ChatGPT subscriptions, and OpenAI-compatible endpoints such +as LiteLLM, Ollama, and vLLM. You can connect more than one inference provider in the same deployment. That lets you mix and match models by provider instead of betting the whole @@ -37,14 +38,34 @@ in the setup wizard and from **Settings > Models**. You stay in control: you can disable or remove any of the added models, add more later, and reconnecting a provider never re-adds models you removed. +### Self-hosted and gateway providers + +LiteLLM, Ollama, and vLLM let you bring an endpoint that you operate or host on +your own infrastructure. Their models are discovered from the configured +endpoint instead of Roomote's recommended catalog. Connect the provider in +**Settings > Models**, then select from the discovered models and choose the +defaults and role mappings that fit your deployment. + +| Provider | Best for | Setup | +| --- | --- | --- | +| [LiteLLM](/providers/inference/litellm) | A single gateway that routes to one or more model vendors | Endpoint URL and required API key | +| [Ollama](/providers/inference/ollama) | Local or private model serving without provider API keys | Endpoint URL | +| [vLLM](/providers/inference/vllm) | Serving OpenAI-compatible models on your own GPU infrastructure | Endpoint URL and optional API key | + +These endpoints must be reachable from the Roomote deployment, not merely from +your laptop or an individual task sandbox. See the provider page for the +expected URL, network, security, and cost behavior. + The recommended set is a single curated list of models that ships with each -Roomote release, so it is predictable for a given version. Every provider -draws from the same list: a provider offers the subset it serves, under its -own model ids, with the same names everywhere. **Available Models** always lists the -full recommended set for every connected provider: recommendations you have -not enabled appear toggled off, and they cannot be deleted while their -provider stays connected — turn a model off to stop using it. To go beyond the -recommended set, add any model by its slug from the add-model field. +Roomote release, so it is predictable for a given version. Every catalog-backed +provider draws from the same list: a provider offers the subset it serves, under +its own model ids, with the same names everywhere. Endpoint providers that +discover models dynamically use the model list returned by their endpoint. +**Available Models** always lists the full recommended set for every connected +catalog-backed provider: recommendations you have not enabled appear toggled +off, and they cannot be deleted while their provider stays connected — turn a +model off to stop using it. To go beyond the recommended set, add any model by +its slug from the add-model field. ### Recommended default models diff --git a/apps/docs/providers/inference/litellm.mdx b/apps/docs/providers/inference/litellm.mdx new file mode 100644 index 000000000..272c88fe0 --- /dev/null +++ b/apps/docs/providers/inference/litellm.mdx @@ -0,0 +1,81 @@ +--- +title: LiteLLM +icon: route +description: Route Roomote model calls through a LiteLLM gateway. +--- + +LiteLLM is an OpenAI-compatible inference gateway. Use it when you want one +deployment-owned endpoint to route Roomote tasks to different model vendors, +apply gateway policies, or centralize model credentials and spend controls. + +## Configure LiteLLM + +In **Settings > Models**, add LiteLLM and provide its endpoint URL and API key. +You can instead manage both values as deployment environment variables: + +```sh +LITELLM_BASE_URL=https://litellm.example.com/v1 +LITELLM_API_KEY=... +``` + +`LITELLM_API_KEY` is required. The endpoint must be reachable from the Roomote +deployment. For a Compose deployment, use a service DNS name such as +`http://litellm:4000/v1` when LiteLLM runs on the same private network. Do not +use `localhost` unless LiteLLM runs in the same network namespace as the +Roomote service that proxies inference requests. + +After saving the provider, Roomote discovers the models exposed by the gateway. +Enable the models you want and select a default coding model and any specialized +roles. Model IDs use the `litellm/` form, where `` is the +name configured by your gateway. + +### LiteLLM proxy flags + +Start LiteLLM with the configuration and network settings appropriate for your +deployment. A typical proxy command supplies a config file and listen port: + +```sh +litellm --config /path/to/config.yaml --port 4000 +``` + +Use LiteLLM's master-key configuration to require the same key supplied as +`LITELLM_API_KEY`, and configure model aliases and pricing in the LiteLLM config +file. Keep the proxy bound to a private interface unless Roomote reaches it +through a protected ingress. + +## Secure the gateway + +Keep LiteLLM private to Roomote whenever possible. Put it on an internal network +or behind a private ingress, require its API key, and use TLS when traffic +crosses an untrusted network. Store `LITELLM_API_KEY` in your deployment secret +manager or as an encrypted Roomote deployment variable, not in an environment's +task variables or repository files. + +Roomote proxies model traffic through its inference gateway, so task sandboxes +do not need direct network access to LiteLLM or the gateway key. Restrict the +gateway key to the model access and spend limits Roomote needs. + +## Cost behavior + +LiteLLM can calculate and return usage costs for requests it routes. When your +LiteLLM model catalog has pricing configured, Roomote can show that +gateway-reported cost alongside task usage. LiteLLM remains the source of truth +for its budgets, rate limits, provider billing, and any model-specific pricing +overrides. + +## Verify setup + +1. save the endpoint URL and API key +2. confirm models appear in **Settings > Models** +3. enable one model and make it the default coding model +4. start a small Roomote task and confirm it completes through the gateway +5. check LiteLLM logs and usage data for the request + +## Common issues + +- **No models appear.** Confirm the endpoint URL includes the LiteLLM `/v1` + API path and that the API key can list models. +- **Tasks cannot reach LiteLLM.** Check DNS, container networking, firewall + rules, and whether the endpoint is reachable from the Roomote deployment. +- **Costs are missing or unexpected.** Check LiteLLM's model pricing and + routing configuration; local gateway settings can override upstream defaults. diff --git a/apps/docs/providers/inference/ollama.mdx b/apps/docs/providers/inference/ollama.mdx new file mode 100644 index 000000000..d1291921d --- /dev/null +++ b/apps/docs/providers/inference/ollama.mdx @@ -0,0 +1,74 @@ +--- +title: Ollama +icon: cpu +description: Use locally or privately served Ollama models with Roomote. +--- + +Ollama serves models on infrastructure you control. Use it for local +development, private deployments, or workloads where you want to keep inference +inside your own network. + +## Configure Ollama + +In **Settings > Models**, add Ollama and provide the endpoint URL. Ollama does +not require an API key. You can also set the endpoint as a deployment variable: + +```sh +OLLAMA_BASE_URL=http://ollama:11434 +``` + +Use the Ollama service root, not its `/v1` path. The local default is +`http://127.0.0.1:11434`; in a container deployment, use a hostname that +Roomote can resolve, such as `http://ollama:11434`. + +After saving the provider, Roomote discovers available Ollama models. Enable the +models you need and select them using `ollama/`, for example +`ollama/qwen3-coder`. + +## Connectivity and security + +The Ollama endpoint must be reachable from the Roomote deployment. It does not +need to be exposed to every Roomote task sandbox. When Roomote and Ollama run in +separate containers, `127.0.0.1` points to the Roomote container, not the +Ollama container; use a shared network and service DNS instead. + +Ollama normally accepts unauthenticated requests. Keep it bound to loopback or +a private network, and do not publish it directly to the internet. If traffic +must leave a trusted network, place a TLS-terminating, authenticated proxy in +front of Ollama and point Roomote at that private or protected endpoint. + +### Ollama listen address + +Ollama listens only where its `OLLAMA_HOST` setting permits. To make it +reachable by another container on a private network, configure a listen address +that accepts the network connection, for example: + +```sh +OLLAMA_HOST=0.0.0.0:11434 ollama serve +``` + +Do not use a public bind address without a protected reverse proxy or equivalent +network controls. Keep `OLLAMA_BASE_URL` pointed at the private service URL. + +## Cost behavior + +Ollama does not charge a per-token provider price for locally served models. +Roomote can record model usage, but it cannot infer your hardware, electricity, +hosting, or capacity costs. Monitor GPU utilization and infrastructure spend +separately when comparing Ollama with hosted providers. + +## Verify setup + +1. pull a model with Ollama and confirm it appears in `ollama list` +2. save `OLLAMA_BASE_URL` or the equivalent Settings value +3. confirm the model appears in **Settings > Models** +4. enable it, make it the default coding model, and start a small task + +## Common issues + +- **The model list is empty.** Confirm Ollama is running, the model is pulled, + and Roomote can reach the configured endpoint. +- **Connection refused in Docker.** Use the Ollama service name instead of + `127.0.0.1`, and confirm both services share a network. +- **Responses are slow or fail under load.** Choose a smaller model, add GPU + capacity, or reduce concurrent task demand. diff --git a/apps/docs/providers/inference/vllm.mdx b/apps/docs/providers/inference/vllm.mdx new file mode 100644 index 000000000..95a418299 --- /dev/null +++ b/apps/docs/providers/inference/vllm.mdx @@ -0,0 +1,76 @@ +--- +title: vLLM +icon: server +description: Serve OpenAI-compatible models on vLLM infrastructure for Roomote tasks. +--- + +vLLM exposes OpenAI-compatible APIs for models that you serve on your own GPU +infrastructure. Use it when you need control over model hosting, capacity, and +network placement while keeping the standard Roomote model workflow. + +## Configure vLLM + +In **Settings > Models**, add vLLM and provide its endpoint URL. If your vLLM +server requires bearer authentication, also provide an API key. You can manage +the same values as deployment environment variables: + +```sh +VLLM_BASE_URL=https://vllm.example.com/v1 +VLLM_API_KEY=... +``` + +`VLLM_API_KEY` is optional only when the vLLM endpoint intentionally accepts +unauthenticated traffic. The endpoint normally includes the `/v1` API path. +After saving it, Roomote discovers the served models. Enable the models you want +and use IDs in the `vllm/` form. + +### vLLM server flags + +Set the served model name to the model ID you want Roomote to discover, and use +vLLM's host, port, and API-key flags to match the endpoint configuration: + +```sh +vllm serve --host 0.0.0.0 --port 8000 --api-key "$VLLM_API_KEY" \ + --served-model-name +``` + +When vLLM is behind a reverse proxy, keep the `/v1` path available and point +`VLLM_BASE_URL` at the proxy's OpenAI-compatible API URL. + +## Connectivity and security + +The vLLM endpoint must be reachable from the Roomote deployment. Place a +same-host service behind a private network or connect remote infrastructure over +TLS and a restricted ingress. A task sandbox does not need direct vLLM access: +Roomote's inference gateway holds the deployment credential and proxies model +requests. + +Require `VLLM_API_KEY` when the endpoint is reachable beyond a tightly +controlled private network. Store the key in your deployment secret manager or +as an encrypted Roomote deployment variable, never in a repository or task +environment. Limit inbound access to Roomote and trusted operators; OpenAI- +compatible inference endpoints are not safe to expose publicly without an +authentication and network boundary. + +## Cost behavior + +vLLM can report token usage, but Roomote cannot calculate the cost of hardware +you operate. Account separately for GPU instances, reserved capacity, +electricity, storage, and idle time. Use task usage to understand demand and +your infrastructure monitoring to set the cost per token or per task. + +## Verify setup + +1. confirm the vLLM server lists its served model at its `/v1/models` endpoint +2. save `VLLM_BASE_URL` and, when required, `VLLM_API_KEY` +3. confirm the model appears in **Settings > Models** +4. enable it, select it for a model role, and start a small task + +## Common issues + +- **No models appear.** Confirm the endpoint includes `/v1` and that Roomote + can call the server's models endpoint. +- **Requests return unauthorized.** Check `VLLM_API_KEY`, its bearer-token + configuration on vLLM, and any reverse-proxy authentication. +- **Tasks are slow or queue.** Check GPU memory, batch and concurrency settings, + and capacity at the vLLM server. diff --git a/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.client.test.tsx b/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.client.test.tsx index 2cc903d4c..087f66f20 100644 --- a/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.client.test.tsx +++ b/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.client.test.tsx @@ -7,6 +7,7 @@ import type { import { fireEvent, render, screen, waitFor } from '@testing-library/react'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import type { SetupModelStatus } from '@roomote/types'; +import { toast } from 'sonner'; const { mutateAsyncMock } = vi.hoisted(() => ({ mutateAsyncMock: vi.fn(), @@ -15,6 +16,7 @@ const { mutateAsyncMock } = vi.hoisted(() => ({ vi.mock('sonner', () => ({ toast: { error: vi.fn(), + success: vi.fn(), }, })); @@ -33,6 +35,14 @@ vi.mock('@/trpc/client', () => ({ queryOptions: () => ({ queryKey: ['chatgptSubscription.status'] }), }, }, + taskModels: { + discoverProviderModels: { + mutationOptions: (options: Record) => options, + }, + qualifyProviderModel: { + mutationOptions: (options: Record) => options, + }, + }, }), })); @@ -149,6 +159,22 @@ function openrouterProviderStatus(): SetupModelStatus['providers'][number] { }; } +function ollamaProviderStatus(): SetupModelStatus['providers'][number] { + return { + id: 'ollama', + label: 'Ollama', + envVarName: 'OLLAMA_BASE_URL', + envVarLabel: 'Endpoint URL', + defaultRoomoteModel: '', + authKind: 'endpoint', + suggestedTaskModels: [], + additionalEnvFields: [], + additionalEnvValues: {}, + runtimeApiKeySatisfied: false, + savedApiKeySatisfied: false, + }; +} + function buildModelSetup( overrides: Partial = {}, ): SetupModelStatus { @@ -246,6 +272,119 @@ describe('StepInferenceProvider configured API key display', () => { fireEvent.focus(input); expect(input).toHaveValue(''); }); + + it('checks an endpoint and saves its recommended qualified model without showing a picker', async () => { + mutateAsyncMock + .mockResolvedValueOnce({ + error: null, + modelCount: 2, + recommendedModels: [{ modelId: 'ollama/qwen3-coder:30b' }], + }) + .mockResolvedValueOnce({ success: true }) + .mockResolvedValueOnce(undefined); + + render( + , + ); + + fireEvent.change(screen.getByPlaceholderText(/Endpoint URL for Ollama/i), { + target: { value: 'http://ollama.example' }, + }); + fireEvent.click(screen.getByRole('button', { name: /continue/i })); + + await waitFor(() => { + expect(mutateAsyncMock).toHaveBeenNthCalledWith(1, { + provider: 'ollama', + baseUrl: 'http://ollama.example', + apiKey: undefined, + }); + }); + expect(mutateAsyncMock).toHaveBeenNthCalledWith(2, { + provider: 'ollama', + modelId: 'ollama/qwen3-coder:30b', + baseUrl: 'http://ollama.example', + apiKey: undefined, + }); + expect(mutateAsyncMock).toHaveBeenNthCalledWith(3, { + provider: 'ollama', + apiKey: 'http://ollama.example', + modelId: 'ollama/qwen3-coder:30b', + }); + expect( + screen.queryByRole('combobox', { name: 'Discovered model' }), + ).not.toBeInTheDocument(); + }); + + it('explains the minimum model requirements when no local model is eligible', async () => { + mutateAsyncMock.mockResolvedValueOnce({ + error: null, + modelCount: 2, + recommendedModels: [], + }); + + render( + , + ); + + fireEvent.change(screen.getByPlaceholderText(/Endpoint URL for Ollama/i), { + target: { value: 'http://ollama.example' }, + }); + fireEvent.click(screen.getByRole('button', { name: /continue/i })); + + await waitFor(() => { + expect(toast.error).toHaveBeenCalledWith( + 'Found 2 models, but none that can power Roomote. It needs tool calling and at least 7B parameters.', + ); + }); + expect(mutateAsyncMock).toHaveBeenCalledTimes(1); + }); + + it('reports a tool-calling failure separately from model eligibility', async () => { + mutateAsyncMock + .mockResolvedValueOnce({ + error: null, + modelCount: 1, + recommendedModels: [{ modelId: 'ollama/qwen3:8b' }], + }) + .mockResolvedValueOnce({ + success: false, + error: 'The provider requires a tool-call parser.', + }); + + render( + , + ); + + fireEvent.change(screen.getByPlaceholderText(/Endpoint URL for Ollama/i), { + target: { value: 'http://ollama.example' }, + }); + fireEvent.click(screen.getByRole('button', { name: /continue/i })); + + await waitFor(() => { + expect(toast.error).toHaveBeenCalledWith( + "Found 1 model that meets Roomote's 7B minimum, but none support the required tool calling. The provider requires a tool-call parser.", + ); + }); + expect(mutateAsyncMock).toHaveBeenCalledTimes(2); + }); }); describe('StepInferenceProvider ChatGPT subscription', () => { @@ -361,7 +500,7 @@ describe('StepInferenceProvider ChatGPT subscription', () => { }); // The mutation's onSuccess handler calls onContinue after invalidating. - const options = mockUseMutation.mock.calls.at(-1)?.[0] as + const options = mockUseMutation.mock.calls[0]?.[0] as | { onSuccess?: () => Promise | void } | undefined; await options?.onSuccess?.(); diff --git a/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.tsx b/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.tsx index 7517c93ee..2ebef530f 100644 --- a/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.tsx +++ b/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.tsx @@ -103,13 +103,25 @@ export function StepInferenceProvider({ }, }), ); + const discoverProviderModels = useMutation( + trpc.taskModels.discoverProviderModels.mutationOptions(), + ); + const qualifyProviderModel = useMutation( + trpc.taskModels.qualifyProviderModel.mutationOptions(), + ); useEffect(() => { setSelectedProvider(modelSetup.preselectedProvider); }, [modelSetup.preselectedProvider]); useEffect(() => { - setApiKey(''); + setApiKey( + selectedProvider === 'ollama' + ? 'http://localhost:11434' + : selectedProvider === 'vllm' + ? 'http://localhost:8000/v1' + : '', + ); setAdditionalEnvValues({}); setEditingSavedValue(false); setIsChatGptDialogOpen(false); @@ -137,6 +149,7 @@ export function StepInferenceProvider({ const isChatGptProvider = selectedProviderStatus?.authKind === 'oauth' && selectedProvider === CHATGPT_SUBSCRIPTION_PROVIDER_ID; + const isEndpointProvider = selectedProviderStatus?.authKind === 'endpoint'; const chatgptConnected = Boolean(modelSetup.chatgptConnected); const hasRuntimeProviderKey = selectedProviderStatus?.runtimeApiKeySatisfied === true; @@ -155,6 +168,7 @@ export function StepInferenceProvider({ [modelSetup.providers], ); const shouldShowSavedValueMask = + !isEndpointProvider && !hasRuntimeProviderKey && hasSavedProviderKey && apiKey.length === 0 && @@ -171,15 +185,75 @@ export function StepInferenceProvider({ ); const isActionDisabled = saveModelConfig.isPending || + discoverProviderModels.isPending || + qualifyProviderModel.isPending || hasMissingRequiredFields || (!canContinueWithoutApiKey && apiKey.trim().length === 0); + const isCheckingEndpoint = + isEndpointProvider && + (discoverProviderModels.isPending || qualifyProviderModel.isPending); const handleContinue = async () => { + let modelId: string | undefined; + let endpointConnectionMessage: string | undefined; + let qualificationError: string | undefined; + const submittedCredential = shouldShowConfiguredMask + ? undefined + : apiKey.trim() || undefined; + + if (isEndpointProvider) { + const provider = selectedProvider as 'ollama' | 'vllm' | 'litellm'; + const connection = { + baseUrl: submittedCredential, + apiKey: + additionalEnvValues[ + `${selectedProvider.toUpperCase()}_API_KEY` + ]?.trim() || undefined, + }; + const discovery = await discoverProviderModels.mutateAsync({ + provider, + baseUrl: connection.baseUrl, + apiKey: connection.apiKey, + }); + if (discovery.error) { + toast.error(discovery.error); + return; + } + + for (const candidate of discovery.recommendedModels) { + const result = await qualifyProviderModel.mutateAsync({ + provider, + modelId: candidate.modelId, + baseUrl: connection.baseUrl, + apiKey: connection.apiKey, + }); + if (result.success) { + modelId = candidate.modelId; + break; + } + qualificationError = result.error; + } + + if (!modelId) { + toast.error( + discovery.recommendedModels.length === 0 + ? `Found ${discovery.modelCount} ${discovery.modelCount === 1 ? 'model' : 'models'}, but none that can power Roomote. It needs tool calling and at least 7B parameters.` + : `Found ${discovery.modelCount} ${discovery.modelCount === 1 ? 'model that meets' : 'models that meet'} Roomote's 7B minimum, but none support the required tool calling. ${qualificationError ?? 'Check the provider tool-calling configuration.'}`, + ); + return; + } + + endpointConnectionMessage = `Connected to ${selectedProviderStatus?.label ?? 'the provider'} and selected ${modelId.replace(`${provider}/`, '')} from ${discovery.modelCount} discovered ${discovery.modelCount === 1 ? 'model' : 'models'}.`; + } await saveModelConfig.mutateAsync({ provider: selectedProvider, - apiKey: apiKey.trim() || undefined, + apiKey: submittedCredential, ...(additionalEnvFields.length > 0 && { additionalEnvValues }), + ...(modelId && { modelId }), }); + if (endpointConnectionMessage) { + toast.success(endpointConnectionMessage); + } }; return ( @@ -199,7 +273,7 @@ export function StepInferenceProvider({ setSelectedProvider(value as SetupModelProviderId) } > - + @@ -213,7 +287,7 @@ export function StepInferenceProvider({ {isChatGptProvider ? null : ( { if (shouldShowSavedValueMask) { @@ -344,9 +418,9 @@ export function StepInferenceProvider({ onClick={() => void handleContinue()} disabled={isActionDisabled} > - {saveModelConfig.isPending ? ( + {saveModelConfig.isPending || isCheckingEndpoint ? ( <> - Saving... + {isCheckingEndpoint ? 'Checking connection...' : 'Saving...'} ) : ( diff --git a/apps/web/src/components/settings/InferenceProviderSection.test.tsx b/apps/web/src/components/settings/InferenceProviderSection.test.tsx index 469cddfdc..eee91b70b 100644 --- a/apps/web/src/components/settings/InferenceProviderSection.test.tsx +++ b/apps/web/src/components/settings/InferenceProviderSection.test.tsx @@ -42,6 +42,9 @@ vi.mock('@/trpc/client', () => ({ deleteProvider: { mutationOptions: () => ({ mutationKey: ['deleteProvider'] }), }, + discoverProviderModels: { + mutationOptions: () => ({ mutationKey: ['discoverProviderModels'] }), + }, }, chatgptSubscription: { status: { @@ -549,4 +552,94 @@ describe('InferenceProviderSection', () => { additionalEnvValues: { AWS_REGION: 'us-west-2' }, }); }); + + it('closes the endpoint dialog after connecting', async () => { + const { providerSetup } = buildProviderSetup(); + providerSetup.providers = [ + { + id: 'ollama', + label: 'Ollama', + envVarName: 'OLLAMA_BASE_URL', + envVarLabel: 'Endpoint URL', + defaultRoomoteModel: '', + authKind: 'endpoint', + suggestedTaskModels: [], + additionalEnvFields: [], + runtimeApiKeySatisfied: false, + savedApiKeySatisfied: false, + additionalEnvValues: {}, + }, + ]; + providerSetupData.current = { providerSetup }; + mutateAsyncMock.mockResolvedValue({ + addedRecommendedModelCount: 0, + addedDiscoveredModelCount: 1, + discoveryError: null, + }); + + renderInferenceProviderSection(); + + fireEvent.click(screen.getByRole('button', { name: /Add provider/ })); + fireEvent.change(screen.getByLabelText('Endpoint URL for Ollama'), { + target: { value: 'http://ollama.example' }, + }); + + await act(async () => { + fireEvent.click(screen.getByRole('button', { name: 'Add' })); + }); + + expect(mutateAsyncMock).toHaveBeenCalledWith({ + provider: 'ollama', + apiKey: 'http://ollama.example', + }); + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + }); + + it('keeps endpoint credentials while provider metadata refreshes', () => { + const { providerSetup } = buildProviderSetup(); + providerSetup.providers = [ + { + id: 'ollama', + label: 'Ollama', + envVarName: 'OLLAMA_BASE_URL', + envVarLabel: 'Endpoint URL', + defaultRoomoteModel: '', + authKind: 'endpoint', + suggestedTaskModels: [], + additionalEnvFields: [], + runtimeApiKeySatisfied: false, + savedApiKeySatisfied: false, + additionalEnvValues: {}, + }, + ]; + providerSetupData.current = { providerSetup }; + + const view = renderInferenceProviderSection(); + + fireEvent.click(screen.getByRole('button', { name: /Add provider/ })); + fireEvent.change(screen.getByLabelText('Endpoint URL for Ollama'), { + target: { value: 'http://127.0.0.1:11434' }, + }); + + const refreshedProviderSetup = { + ...providerSetup, + providers: providerSetup.providers.map((provider) => ({ ...provider })), + }; + const { connectedProviders, availableProviders } = splitInferenceProviders( + refreshedProviderSetup, + ); + + view.rerender( + , + ); + + expect(screen.getByLabelText('Endpoint URL for Ollama')).toHaveValue( + 'http://127.0.0.1:11434', + ); + }); }); diff --git a/apps/web/src/components/settings/InferenceProviderSection.tsx b/apps/web/src/components/settings/InferenceProviderSection.tsx index efb9b00c6..f6be7f064 100644 --- a/apps/web/src/components/settings/InferenceProviderSection.tsx +++ b/apps/web/src/components/settings/InferenceProviderSection.tsx @@ -58,7 +58,7 @@ type InferenceProviderSectionProps = { }; type ProviderCredentialsDialogState = - | { mode: 'add' } + | { mode: 'add'; providerId?: SetupModelProviderId } | { mode: 'edit'; providerId: SetupModelProviderId }; function getInitialAdditionalEnvValues( @@ -88,7 +88,11 @@ function ConnectedProviderRow({ const runtimeKeyLabel = provider.envVarName ? `${provider.label} API key is managed by ${provider.envVarName}` : `${provider.label} API key is managed by an environment variable`; - const inputValue = MASKED_VALUE; + const inputValue = + provider.authKind === 'endpoint' + ? (provider.additionalEnvValues[provider.envVarName ?? ''] ?? + 'Configured endpoint') + : MASKED_VALUE; return (
@@ -183,18 +187,30 @@ function ProviderCredentialsDialog({ const [providerSelectOpen, setProviderSelectOpen] = useState(false); const [additionalEnvValues, setAdditionalEnvValues] = useState< Record - >({}); + >(() => getInitialAdditionalEnvValues(providers[0] ?? null)); useEffect(() => { if (!open) { return; } + if ( + selectedProviderId && + providers.some((provider) => provider.id === selectedProviderId) + ) { + return; + } + + // Saving an endpoint provider refreshes its metadata and replaces the + // provider objects passed to this dialog. Keep the current selection and + // form values when that happens; otherwise the refresh clears the endpoint + // while model discovery is in flight and repeatedly reinitializes the + // dialog. Only initialize values when the selection is no longer valid. const provider = providers[0] ?? null; setSelectedProviderId(provider?.id ?? null); setApiKey(''); setAdditionalEnvValues(getInitialAdditionalEnvValues(provider)); - }, [open, providers]); + }, [open, providers, selectedProviderId]); useEffect(() => { if (!open || mode !== 'add') { @@ -323,7 +339,7 @@ function ProviderCredentialsDialog({
setApiKey(event.target.value)} @@ -555,18 +571,24 @@ export function InferenceProviderSection({ onSuccess: async (result, variables) => { const providerLabel = getModelProviderLabel(variables.provider); const addedModelCount = result.addedRecommendedModelCount; + const addedDiscoveredModelCount = result.addedDiscoveredModelCount; toast.success( - addedModelCount > 0 - ? `Saved the ${providerLabel} API key and added ${addedModelCount} recommended ${addedModelCount === 1 ? 'model' : 'models'}.` - : `Saved the ${providerLabel} API key.`, + addedDiscoveredModelCount > 0 + ? `Saved the ${providerLabel} API key and made ${addedDiscoveredModelCount} discovered ${addedDiscoveredModelCount === 1 ? 'model' : 'models'} available.` + : addedModelCount > 0 + ? `Saved the ${providerLabel} API key and added ${addedModelCount} recommended ${addedModelCount === 1 ? 'model' : 'models'}.` + : `Saved the ${providerLabel} API key.`, ); setProviderDialog(null); + if (result.discoveryError) { + toast.error(result.discoveryError); + } await Promise.all([ queryClient.invalidateQueries({ queryKey: trpc.taskModels.providerSetup.queryKey(), }), - ...(addedModelCount > 0 + ...(addedModelCount > 0 || addedDiscoveredModelCount > 0 ? [ queryClient.invalidateQueries({ queryKey: trpc.taskModels.get.queryKey(), @@ -578,7 +600,7 @@ export function InferenceProviderSection({ : []), ]); - if (addedModelCount > 0) { + if (addedModelCount > 0 || addedDiscoveredModelCount > 0) { onRecommendedModelsAdded?.(); } }, @@ -687,7 +709,7 @@ export function InferenceProviderSection({ const addableProviders = availableProviders.filter((provider) => provider.id === CHATGPT_SUBSCRIPTION_PROVIDER_ID ? !chatgptHasRecord - : provider.authKind === 'api-key', + : provider.authKind === 'api-key' || provider.authKind === 'endpoint', ); const sortedApiKeyConnectedProviders = useMemo( () => @@ -709,6 +731,14 @@ export function InferenceProviderSection({ } if (providerDialog.mode === 'add') { + if (providerDialog.providerId) { + const savedProvider = providerSetup?.providers.find( + (provider) => provider.id === providerDialog.providerId, + ); + + return savedProvider ? [savedProvider] : []; + } + return sortedAddableProviders; } @@ -717,7 +747,12 @@ export function InferenceProviderSection({ ); return editProvider ? [editProvider] : []; - }, [providerDialog, sortedAddableProviders, sortedApiKeyConnectedProviders]); + }, [ + providerDialog, + providerSetup?.providers, + sortedAddableProviders, + sortedApiKeyConnectedProviders, + ]); const deleteProviderStatus = useMemo( () => deleteProviderId diff --git a/apps/web/src/trpc/commands/setup-new/index.ts b/apps/web/src/trpc/commands/setup-new/index.ts index d54762a25..a5dd7b215 100644 --- a/apps/web/src/trpc/commands/setup-new/index.ts +++ b/apps/web/src/trpc/commands/setup-new/index.ts @@ -48,11 +48,13 @@ import { } from '@roomote/sdk/server'; import { buildRecommendedDeploymentModelConfig, + buildTaskModelOption, buildSetupAuthStatus, buildSetupComputeStatus, buildSetupModelStatus, buildSetupSourceControlStatus, collectSetupModelProviderCredentialValues, + createEmptyDeploymentModelConfig, createEmptySetupNewState, RunStatus, TaskPayloadKind, @@ -64,12 +66,15 @@ import { getSetupComputeProvider, getComputeFieldValidationError, getSetupModelProvider, + getSetupModelProviderAdditionalEnvFields, + SETUP_MODEL_PROVIDER_CATALOG, isAutoProvisionedComputeArtifactField, isComputeCredentialField, isComputeInfrastructureField, isConfiguredEnvValue, isExitedRunStatus, isRequiredComputeField, + normalizeTaskModelSettings, NON_SECRET_AUTH_ENV_VAR_NAMES, NON_SECRET_COMPUTE_ENV_VAR_NAMES, NON_SECRET_SOURCE_CONTROL_ENV_VAR_NAMES, @@ -1313,6 +1318,7 @@ export async function getSetupNewStatusCommand(auth: UserAuthSuccess) { persistedRuntimeComputeConfig, envVarNames, nonSecretAuthEnvValues, + nonSecretModelEnvValues, nonSecretComputeEnvValues, nonSecretSourceControlEnvValues, chatgptConnected, @@ -1323,6 +1329,16 @@ export async function getSetupNewStatusCommand(auth: UserAuthSuccess) { getPersistedRuntimeComputeConfig(), getPersistedEnvironmentVariableNames(), getPersistedEnvironmentVariableValues([...NON_SECRET_AUTH_ENV_VAR_NAMES]), + getPersistedEnvironmentVariableValues( + SETUP_MODEL_PROVIDER_CATALOG.flatMap((provider) => [ + ...(provider.authKind === 'endpoint' && provider.envVarName + ? [provider.envVarName] + : []), + ...getSetupModelProviderAdditionalEnvFields(provider) + .filter((field) => !field.secret) + .map((field) => field.envVarName), + ]), + ), getPersistedEnvironmentVariableValues([ ...NON_SECRET_COMPUTE_ENV_VAR_NAMES, ]), @@ -1415,6 +1431,7 @@ export async function getSetupNewStatusCommand(auth: UserAuthSuccess) { runtimeEnv: process.env, persistedModelConfig: persistedRuntimeModelConfig, persistedEnvVarNames: envVarNames, + persistedEnvVarValues: nonSecretModelEnvValues, selectedProvider: setupNewState.modelProvider, chatgptConnected, }); @@ -1483,6 +1500,7 @@ export async function saveSetupNewModelConfigCommand( provider: SetupModelProviderId; apiKey?: string; additionalEnvValues?: Record; + modelId?: string; }, ) { assertAdmin(auth); @@ -1558,7 +1576,20 @@ export async function saveSetupNewModelConfigCommand( // Connecting a provider applies its recommended per-role model defaults: // the provider's default coding model plus any recommended helper, // vision, code review, explore, and planning models. - const runtimeModelConfig = buildRecommendedDeploymentModelConfig(provider); + const selectedDynamicModel = input.modelId?.trim(); + + if (provider.dynamicModels && !selectedDynamicModel) { + throw new Error( + `Choose a discovered ${provider.label} model to continue.`, + ); + } + + const runtimeModelConfig = provider.dynamicModels + ? { + ...createEmptyDeploymentModelConfig(), + roomoteModel: selectedDynamicModel!, + } + : buildRecommendedDeploymentModelConfig(provider); // Mirror the models settings page: connecting a provider the deployment // has no models for yet auto-adds its recommended models so the first @@ -1576,13 +1607,35 @@ export async function saveSetupNewModelConfigCommand( persistedTaskModelSettings, connectedProviderIds, }); + const dynamicModelSettings = provider.dynamicModels + ? (() => { + const model = buildTaskModelOption({ + id: selectedDynamicModel!, + displayName: selectedDynamicModel!.split('/').at(-1)!, + }); + const current = normalizeTaskModelSettings( + persistedTaskModelSettings, + ); + + return normalizeTaskModelSettings({ + models: [ + ...(current.models ?? []).filter((item) => item.id !== model.id), + model, + ], + allowedModelIds: [...current.allowedModelIds, model.id], + defaultModelId: model.id, + }); + })() + : null; await Promise.all([ savePersistedSetupNewState(setupNewState, tx), savePersistedRuntimeModelConfig(runtimeModelConfig, tx), - ...(autoAdd - ? [savePersistedTaskModelSettings(autoAdd.taskModelSettings, tx)] - : []), + ...(dynamicModelSettings + ? [savePersistedTaskModelSettings(dynamicModelSettings, tx)] + : autoAdd + ? [savePersistedTaskModelSettings(autoAdd.taskModelSettings, tx)] + : []), ]); return { diff --git a/apps/web/src/trpc/commands/task-models/auto-add-models.ts b/apps/web/src/trpc/commands/task-models/auto-add-models.ts index 8e9595c96..1d53c48f5 100644 --- a/apps/web/src/trpc/commands/task-models/auto-add-models.ts +++ b/apps/web/src/trpc/commands/task-models/auto-add-models.ts @@ -128,6 +128,12 @@ export function buildAutoAddedTaskModelSettings(options: { taskModelSettings: TaskModelSettings; addedModels: TaskModelOption[]; } | null { + // Endpoint-backed providers are populated by explicit discovery. Never seed + // a guessed model id when the endpoint itself is the source of truth. + if (options.provider.dynamicModels) { + return null; + } + const parsed = taskModelSettingsSchema.safeParse( options.persistedTaskModelSettings, ); diff --git a/apps/web/src/trpc/commands/task-models/index.test.ts b/apps/web/src/trpc/commands/task-models/index.test.ts index 1cc080b47..d4f7d3739 100644 --- a/apps/web/src/trpc/commands/task-models/index.test.ts +++ b/apps/web/src/trpc/commands/task-models/index.test.ts @@ -68,7 +68,10 @@ import { getTaskModelProviderSetupCommand, getTaskModelSettingsCommand, deleteTaskModelProviderCommand, + discoverProviderModelsCommand, + getRecommendedLocalProviderModels, lookupTaskModelCommand, + qualifyProviderModelCommand, refreshTaskModelMetadataCommand, saveTaskModelProviderCommand, updateTaskModelSettingsCommand, @@ -84,6 +87,8 @@ const PROVIDER_ENV_VAR_NAMES = [ 'AWS_BEARER_TOKEN_BEDROCK', 'AWS_REGION', 'GEMINI_API_KEY', + 'OLLAMA_BASE_URL', + 'VLLM_BASE_URL', 'R_MODEL', ] as const; @@ -112,6 +117,55 @@ function buildMockAuth( } as UserAuthSuccess; } +describe('getRecommendedLocalProviderModels', () => { + it('prefers capable coding models and excludes tiny or specialized models', () => { + const recommended = getRecommendedLocalProviderModels([ + { + modelId: 'ollama/tinyllama:1.1b', + displayName: 'tinyllama:1.1b', + family: null, + metadata: null, + }, + { + modelId: 'ollama/nomic-embed-text', + displayName: 'nomic-embed-text', + family: null, + metadata: null, + }, + { + modelId: 'ollama/llama3.3:70b', + displayName: 'llama3.3:70b', + family: null, + metadata: null, + }, + { + modelId: 'ollama/qwen3-coder:30b', + displayName: 'qwen3-coder:30b', + family: null, + metadata: null, + }, + ]); + + expect(recommended.map((model) => model.modelId)).toEqual([ + 'ollama/qwen3-coder:30b', + 'ollama/llama3.3:70b', + ]); + }); + + it('does not automatically choose unknown local model aliases', () => { + expect( + getRecommendedLocalProviderModels([ + { + modelId: 'litellm/team-default', + displayName: 'team-default', + family: null, + metadata: null, + }, + ]), + ).toEqual([]); + }); +}); + describe('lookupTaskModelCommand', () => { // Settings reads now depend on which provider env keys are configured // (recommended models of connected providers join the catalog), so clear @@ -146,6 +200,7 @@ describe('lookupTaskModelCommand', () => { delete process.env.R_EXPLORE_MODEL_REASONING_EFFORT; delete process.env.R_PLANNING_MODEL_REASONING_EFFORT; mockIsChatGptSubscriptionConnected.mockResolvedValue(false); + mockGetPersistedEnvironmentVariableValues.mockResolvedValue({}); mockFindDeploymentSettings.mockImplementation(async (options) => { const columns = (options as { columns?: Record }) ?.columns; @@ -254,6 +309,199 @@ describe('lookupTaskModelCommand', () => { expect(fetchMock).not.toHaveBeenCalled(); }); + it('discovers Ollama models from /api/tags and prefixes their IDs', async () => { + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ models: [{ name: 'qwen3:8b' }] }), { + headers: { 'content-type': 'application/json' }, + }), + ); + + await expect( + discoverProviderModelsCommand(buildMockAuth(), { + provider: 'ollama', + baseUrl: 'http://ollama.example', + }), + ).resolves.toMatchObject({ + error: null, + modelCount: 1, + recommendedModels: [{ modelId: 'ollama/qwen3:8b' }], + models: [ + { + modelId: 'ollama/qwen3:8b', + displayName: 'qwen3:8b', + }, + ], + }); + expect(fetchMock).toHaveBeenCalledWith( + 'http://ollama.example/api/tags', + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + }); + + it('falls back to the OpenAI models endpoint when Ollama tags are unavailable', async () => { + fetchMock + .mockResolvedValueOnce(new Response(null, { status: 404 })) + .mockResolvedValueOnce( + new Response(JSON.stringify({ data: [{ id: 'llama3.3' }] }), { + headers: { 'content-type': 'application/json' }, + }), + ); + + await expect( + discoverProviderModelsCommand(buildMockAuth(), { + provider: 'ollama', + baseUrl: 'http://ollama.example/v1', + }), + ).resolves.toMatchObject({ + error: null, + models: [{ modelId: 'ollama/llama3.3' }], + }); + expect(fetchMock).toHaveBeenNthCalledWith( + 2, + 'http://ollama.example/v1/models', + expect.anything(), + ); + }); + + it('uses saved LiteLLM credentials and metadata when discovering models', async () => { + mockGetPersistedEnvironmentVariableValues.mockResolvedValue({ + LITELLM_BASE_URL: 'https://litellm.example/v1', + LITELLM_API_KEY: 'saved-key', + }); + fetchMock + .mockResolvedValueOnce( + new Response(JSON.stringify({ data: [{ id: 'azure/gpt-4o' }] }), { + headers: { 'content-type': 'application/json' }, + }), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + data: [ + { + model_name: 'azure/gpt-4o', + model_info: { max_input_tokens: 128_000 }, + }, + ], + }), + { headers: { 'content-type': 'application/json' } }, + ), + ); + + await expect( + discoverProviderModelsCommand(buildMockAuth(), { provider: 'litellm' }), + ).resolves.toMatchObject({ + models: [ + { + modelId: 'litellm/azure/gpt-4o', + metadata: expect.objectContaining({ contextWindow: 128_000 }), + }, + ], + }); + expect(fetchMock).toHaveBeenCalledWith( + 'https://litellm.example/v1/models', + expect.objectContaining({ + headers: { Authorization: 'Bearer saved-key' }, + }), + ); + expect(fetchMock).toHaveBeenNthCalledWith( + 2, + 'https://litellm.example/model/info', + expect.objectContaining({ + headers: { Authorization: 'Bearer saved-key' }, + }), + ); + }); + + it('qualifies a provider model with a streaming tool request', async () => { + fetchMock.mockResolvedValue( + new Response( + 'data: {"choices":[{"delta":{"tool_calls":[{"function":{"name":"ping"}}]}}]}\n\n', + { + headers: { 'content-type': 'text/event-stream' }, + }, + ), + ); + + await expect( + qualifyProviderModelCommand(buildMockAuth(), { + provider: 'vllm', + baseUrl: 'https://vllm.example/v1', + apiKey: 'submitted-key', + modelId: 'vllm/qwen3', + }), + ).resolves.toEqual({ success: true }); + expect(fetchMock).toHaveBeenCalledWith( + 'https://vllm.example/v1/chat/completions', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + Authorization: 'Bearer submitted-key', + }), + body: expect.stringContaining('"stream":true'), + }), + ); + }); + + it('rejects streams that do not call the qualification tool', async () => { + fetchMock.mockResolvedValue( + new Response('data: {"choices":[{"delta":{"content":"pong"}}]}\n\n', { + headers: { 'content-type': 'text/event-stream' }, + }), + ); + + await expect( + qualifyProviderModelCommand(buildMockAuth(), { + provider: 'ollama', + baseUrl: 'http://ollama.example', + modelId: 'ollama/qwen3', + }), + ).resolves.toEqual({ + success: false, + error: expect.stringContaining('did not call the required tool'), + }); + }); + + it('returns provider compatibility details from qualification errors', async () => { + fetchMock.mockResolvedValue( + new Response( + JSON.stringify({ detail: 'tools are unsupported for this model' }), + { + status: 422, + headers: { 'content-type': 'application/json' }, + }, + ), + ); + + await expect( + qualifyProviderModelCommand(buildMockAuth(), { + provider: 'vllm', + baseUrl: 'https://vllm.example', + modelId: 'vllm/qwen3', + }), + ).resolves.toEqual({ + success: false, + error: expect.stringContaining('tools are unsupported for this model'), + }); + }); + + it('uses discovery data when looking up a local provider model', async () => { + mockGetPersistedEnvironmentVariableValues.mockResolvedValue({ + VLLM_BASE_URL: 'https://vllm.example/v1', + }); + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ data: [{ id: 'llama3.3' }] }), { + headers: { 'content-type': 'application/json' }, + }), + ); + + await expect( + lookupTaskModelCommand(buildMockAuth(), { + modelId: 'vllm/llama3.3', + }), + ).resolves.toMatchObject({ modelId: 'vllm/llama3.3' }); + }); + it.each(['google-vertex/gemini-3.5-flash', 'mistral/mistral-large-latest'])( 'rejects disabled direct-provider model id %s', async (modelId) => { @@ -438,101 +686,57 @@ describe('lookupTaskModelCommand', () => { it('accepts shorthand default model IDs when they normalize to an enabled model', async () => { const auth = buildMockAuth(); + mockGetPersistedEnvironmentVariableNames.mockResolvedValue([ + 'OPENROUTER_API_KEY', + ]); - await expect( - updateTaskModelSettingsCommand(auth, { - models: [ - { - id: 'z-ai/glm-5.6', - displayName: 'GLM 5.6', - family: 'GLM', - }, - ], - allowedModelIds: ['z-ai/glm-5.6'], - defaultModelId: 'z-ai/glm-5.6', - helperModelId: null, - visionModelId: null, - codeReviewModelId: null, - planningModelId: null, - codingModelReasoningEffort: null, - helperModelReasoningEffort: null, - visionModelReasoningEffort: null, - codeReviewModelReasoningEffort: null, - planningModelReasoningEffort: null, - }), - ).resolves.toEqual({ + const result = await updateTaskModelSettingsCommand(auth, { + models: [ + { + id: 'z-ai/glm-5.6', + displayName: 'GLM 5.6', + family: 'GLM', + }, + ], + allowedModelIds: ['z-ai/glm-5.6'], + defaultModelId: 'z-ai/glm-5.6', + helperModelId: null, + visionModelId: null, + codeReviewModelId: null, + planningModelId: null, + codingModelReasoningEffort: null, + helperModelReasoningEffort: null, + visionModelReasoningEffort: null, + codeReviewModelReasoningEffort: null, + planningModelReasoningEffort: null, + }); + + if (!result.success) { + throw new Error('Expected the model settings update to succeed.'); + } + + expect(result).toMatchObject({ success: true, settings: { defaultModelId: 'openrouter/z-ai/glm-5.6', - models: [ - { - id: 'openrouter/z-ai/glm-5.6', - displayName: 'GLM 5.6', - family: 'GLM', - metadata: null, - enabled: true, - isDefault: true, - }, - ], runtimeModels: { codingModel: { effectiveModelId: 'openrouter/z-ai/glm-5.6', persistedModelId: 'openrouter/z-ai/glm-5.6', source: 'database', - managedByEnv: false, - reasoningEffort: null, - reasoningManagedByEnv: false, - }, - helperModel: { - effectiveModelId: null, - persistedModelId: null, - source: 'same-as-coding', - managedByEnv: false, - reasoningEffort: null, - reasoningManagedByEnv: false, - }, - visionModel: { - effectiveModelId: null, - persistedModelId: null, - source: 'same-as-coding', - managedByEnv: false, - reasoningEffort: null, - reasoningManagedByEnv: false, - }, - codeReviewModel: { - effectiveModelId: null, - persistedModelId: null, - source: 'same-as-coding', - managedByEnv: false, - reasoningEffort: null, - reasoningManagedByEnv: false, - }, - exploreModel: { - effectiveModelId: null, - persistedModelId: null, - source: 'same-as-coding', - managedByEnv: false, - reasoningEffort: null, - reasoningManagedByEnv: false, - }, - planningModel: { - effectiveModelId: null, - persistedModelId: null, - source: 'same-as-coding', - managedByEnv: false, - reasoningEffort: null, - reasoningManagedByEnv: false, }, }, - helperModelOptions: [ - { - id: 'openrouter/z-ai/glm-5.6', - displayName: 'GLM 5.6', - family: 'GLM', - }, - ], }, }); + expect(result.settings.models).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'openrouter/z-ai/glm-5.6', + enabled: true, + isDefault: true, + }), + ]), + ); expect(mockInsertDeploymentSettings).toHaveBeenCalled(); expect(mockUpdateDeploymentSettings).toHaveBeenCalledWith( @@ -1222,6 +1426,35 @@ describe('task model provider commands', () => { ); }); + it('hides persisted OpenRouter models when only vLLM is connected', async () => { + mockGetPersistedEnvironmentVariableNames.mockResolvedValue([ + 'VLLM_BASE_URL', + ]); + mockFindDeploymentSettings.mockResolvedValue({ + taskModelSettings: { + models: [ + { + id: 'openrouter/openai/gpt-5.6-terra', + displayName: 'GPT 5.6 Terra', + family: 'GPT', + }, + { + id: 'vllm/qwen3:8b', + displayName: 'Qwen 3 8B', + family: 'Qwen', + }, + ], + allowedModelIds: ['openrouter/openai/gpt-5.6-terra', 'vllm/qwen3:8b'], + defaultModelId: 'openrouter/openai/gpt-5.6-terra', + }, + runtimeModelConfig: null, + }); + + const result = await getTaskModelSettingsCommand(buildMockAuth()); + + expect(result.models.map((model) => model.id)).toEqual(['vllm/qwen3:8b']); + }); + it('preselects the saved provider choice and reports saved API keys', async () => { mockGetPersistedEnvironmentVariableNames.mockResolvedValue([ 'ANTHROPIC_API_KEY', @@ -1245,6 +1478,9 @@ describe('task model provider commands', () => { mockGetPersistedEnvironmentVariableNames.mockResolvedValue([ 'AWS_BEARER_TOKEN_BEDROCK', 'AWS_REGION', + 'LITELLM_BASE_URL', + 'OLLAMA_BASE_URL', + 'VLLM_BASE_URL', ]); mockGetPersistedEnvironmentVariableValues.mockResolvedValue({ AWS_REGION: 'us-west-2', @@ -1254,6 +1490,9 @@ describe('task model provider commands', () => { expect(mockGetPersistedEnvironmentVariableValues).toHaveBeenCalledWith([ 'AWS_REGION', + 'LITELLM_BASE_URL', + 'OLLAMA_BASE_URL', + 'VLLM_BASE_URL', ]); expect( result.providerSetup.providers.find( diff --git a/apps/web/src/trpc/commands/task-models/index.ts b/apps/web/src/trpc/commands/task-models/index.ts index e920bb599..f91983e4f 100644 --- a/apps/web/src/trpc/commands/task-models/index.ts +++ b/apps/web/src/trpc/commands/task-models/index.ts @@ -64,6 +64,63 @@ import type { UserAuthSuccess } from '@/types'; const DEFAULT_DEPLOYMENT_ID = 'default'; const MODEL_METADATA_FETCH_TIMEOUT_MS = 10_000; +const LOCAL_PROVIDER_REQUEST_TIMEOUT_MS = 15_000; + +const LOCAL_TASK_MODEL_PROVIDER_IDS = ['ollama', 'vllm', 'litellm'] as const; +type LocalTaskModelProviderId = (typeof LOCAL_TASK_MODEL_PROVIDER_IDS)[number]; + +type LocalProviderConnectionInput = { + baseUrl?: string; + apiKey?: string; +}; + +type LocalProviderConnection = { + baseUrl: string; + apiKey: string | null; +}; + +type LocalProviderModelResponse = { + data?: Array<{ + id?: string; + name?: string; + model?: string; + model_name?: string; + model_info?: Record; + }>; + models?: Array<{ + id?: string; + name?: string; + model?: string; + model_name?: string; + model_info?: Record; + }>; + model_info?: Record>; +}; + +const LOCAL_MODEL_RECOMMENDATION_FAMILIES = [ + { pattern: /qwen(?:[\d.-]*)(?:[-_:]?(?:coder|code))?/i, score: 100 }, + { pattern: /devstral/i, score: 98 }, + { pattern: /gpt-oss/i, score: 96 }, + { pattern: /glm/i, score: 92 }, + { pattern: /mistral/i, score: 88 }, + { pattern: /deepseek/i, score: 86 }, + { pattern: /llama/i, score: 82 }, + { pattern: /gemma/i, score: 76 }, +] as const; + +const UNSUITABLE_LOCAL_MODEL_PATTERN = + /(?:^|[-_:/.])(tiny|embed(?:ding)?|rerank(?:er)?|guard|moderation|vision|vl|ocr|whisper|tts|nomic|all-minilm)(?:$|[-_:/.])/i; +const LOCAL_MODEL_PARAMETER_COUNT_PATTERN = + /(?:^|[-_:/.])(\d+(?:\.\d+)?)b(?:$|[-_:/.])/i; + +const LOCAL_PROVIDER_CONNECTION_ENV: Record< + LocalTaskModelProviderId, + { baseUrl: string; apiKey?: string } +> = { + ollama: { baseUrl: 'OLLAMA_BASE_URL' }, + vllm: { baseUrl: 'VLLM_BASE_URL', apiKey: 'VLLM_API_KEY' }, + litellm: { baseUrl: 'LITELLM_BASE_URL', apiKey: 'LITELLM_API_KEY' }, +}; function assertAdmin(auth: UserAuthSuccess): asserts auth is UserAuthSuccess { if (!auth.isAdmin) { @@ -291,13 +348,20 @@ export async function getTaskModelSettingsCommand( // The Available Models list always shows the full recommended set for // every connected provider; entries that are not persisted yet render // disabled until an operator enables them. + const connectedProviderIds = collectConnectedTaskModelProviderIds({ + runtimeEnv: process.env, + persistedEnvVarNames, + chatgptConnected, + }); + // A provider must be connected before its models can be selected. This also + // removes stale rows created by the old implicit OpenRouter default catalog. const catalog = appendRecommendedTaskModels({ - models: getTaskModelCatalog(settings), - connectedProviderIds: collectConnectedTaskModelProviderIds({ - runtimeEnv: process.env, - persistedEnvVarNames, - chatgptConnected, + models: getTaskModelCatalog(settings).filter((model) => { + const providerId = getTaskModelProviderId(model.id); + + return providerId !== null && connectedProviderIds.has(providerId); }), + connectedProviderIds, }); return { @@ -371,11 +435,14 @@ export async function getTaskModelProviderSetupCommand( getDeploymentRuntimeModelConfig(), getPersistedEnvironmentVariableNames(), getPersistedEnvironmentVariableValues( - SETUP_MODEL_PROVIDER_CATALOG.flatMap((provider) => - getSetupModelProviderAdditionalEnvFields(provider) + SETUP_MODEL_PROVIDER_CATALOG.flatMap((provider) => [ + ...(provider.authKind === 'endpoint' && provider.envVarName + ? [provider.envVarName] + : []), + ...getSetupModelProviderAdditionalEnvFields(provider) .filter((field) => !field.secret) .map((field) => field.envVarName), - ), + ]), ), getDeploymentSetupNewState(), isChatGptSubscriptionConnected(), @@ -414,6 +481,8 @@ export async function saveTaskModelProviderCommand( ): Promise<{ providerSetup: SetupModelStatus; addedRecommendedModelCount: number; + addedDiscoveredModelCount: number; + discoveryError: string | null; }> { assertAdmin(auth); @@ -511,9 +580,85 @@ export async function saveTaskModelProviderCommand( }); }); + let addedDiscoveredModelCount = 0; + let discoveryError: string | null = null; + + if (provider.dynamicModels) { + const discovery = await discoverProviderModelsCommand(auth, { + provider: provider.id as LocalTaskModelProviderId, + }); + discoveryError = discovery.error; + + if (!discovery.error && discovery.models.length > 0) { + await db.transaction(async (tx) => { + const persistedTaskModels = await getPersistedRawTaskModelSettings(tx); + const currentSettings = normalizeTaskModelSettings(persistedTaskModels); + const modelsById = new Map( + (currentSettings.models ?? []).map((model) => [model.id, model]), + ); + + for (const model of discovery.models) { + if (modelsById.has(model.modelId)) { + continue; + } + + modelsById.set( + model.modelId, + buildTaskModelOption({ + id: model.modelId, + displayName: model.displayName ?? model.modelId, + family: model.family ?? undefined, + metadata: model.metadata, + }), + ); + addedDiscoveredModelCount += 1; + } + + if (addedDiscoveredModelCount === 0) { + return; + } + + const models = [...modelsById.values()]; + await tx + .insert(deploymentSettings) + .values({ + id: DEFAULT_DEPLOYMENT_ID, + taskModelSettings: normalizeTaskModelSettings({ + ...currentSettings, + models, + allowedModelIds: [ + ...new Set([ + ...currentSettings.allowedModelIds, + ...discovery.models.map((model) => model.modelId), + ]), + ], + }), + }) + .onConflictDoUpdate({ + target: deploymentSettings.id, + set: { + taskModelSettings: normalizeTaskModelSettings({ + ...currentSettings, + models, + allowedModelIds: [ + ...new Set([ + ...currentSettings.allowedModelIds, + ...discovery.models.map((model) => model.modelId), + ]), + ], + }), + updatedAt: new Date(), + }, + }); + }); + } + } + return { ...(await getTaskModelProviderSetupCommand(auth)), addedRecommendedModelCount, + addedDiscoveredModelCount, + discoveryError, }; } @@ -1088,6 +1233,442 @@ type TaskModelLookupResult = { metadata: TaskModelMetadata | null; }; +function getLocalModelRecommendationScore(model: TaskModelLookupResult) { + const name = `${model.modelId} ${model.displayName ?? ''}`.toLowerCase(); + + if (UNSUITABLE_LOCAL_MODEL_PATTERN.test(name)) { + return null; + } + + const family = LOCAL_MODEL_RECOMMENDATION_FAMILIES.find(({ pattern }) => + pattern.test(name), + ); + if (!family) { + return null; + } + + const parameterCount = Number( + LOCAL_MODEL_PARAMETER_COUNT_PATTERN.exec(name)?.[1] ?? 0, + ); + if (parameterCount > 0 && parameterCount < 7) { + return null; + } + + const codingBonus = /(?:coder|code)/i.test(name) ? 8 : 0; + return family.score + codingBonus + Math.min(parameterCount, 100) / 100; +} + +/** + * Endpoint providers advertise their installed models rather than a stable + * catalog. Keep the automatic choice deliberately conservative: only known + * general-purpose families are eligible, and small or specialized models are + * left for an operator to enable manually from Models settings. + */ +export function getRecommendedLocalProviderModels( + models: readonly TaskModelLookupResult[], +) { + return models + .flatMap((model) => { + const score = getLocalModelRecommendationScore(model); + return score === null ? [] : [{ model, score }]; + }) + .sort( + (left, right) => + right.score - left.score || + left.model.modelId.localeCompare(right.model.modelId), + ) + .map(({ model }) => model); +} + +type LocalProviderDiscoveryResult = { + models: TaskModelLookupResult[]; + modelCount: number; + recommendedModels: TaskModelLookupResult[]; + error: string | null; +}; + +function getLocalProviderBaseUrl(baseUrl: string, path: string) { + const normalized = baseUrl.replace(/\/+$/u, ''); + const withoutV1 = normalized.endsWith('/v1') + ? normalized.slice(0, -'/v1'.length) + : normalized; + + return path.startsWith('/v1/') + ? `${normalized.endsWith('/v1') ? normalized : `${normalized}/v1`}${path.slice('/v1'.length)}` + : `${withoutV1}${path}`; +} + +function getLocalProviderError( + provider: LocalTaskModelProviderId, + response: Response, +) { + const label = + provider === 'vllm' + ? 'vLLM' + : provider === 'litellm' + ? 'LiteLLM' + : 'Ollama'; + + if (response.status === 401 || response.status === 403) { + return `${label} rejected the API key. Check the saved credentials.`; + } + if (response.status === 404) { + return `${label} did not recognize this endpoint. Check the endpoint URL and API compatibility.`; + } + if (response.status === 429) { + return `${label} is rate limiting requests. Try again shortly.`; + } + if (response.status >= 500) { + return `${label} returned a server error (${response.status}). Check that the provider is healthy.`; + } + return `${label} returned HTTP ${response.status}.`; +} + +function getLocalProviderNetworkError(provider: LocalTaskModelProviderId) { + const label = + provider === 'vllm' + ? 'vLLM' + : provider === 'litellm' + ? 'LiteLLM' + : 'Ollama'; + return `Could not reach ${label}. Check the endpoint URL and network access from Roomote.`; +} + +async function getQualificationError( + provider: LocalTaskModelProviderId, + response: Response, +) { + const baseError = getLocalProviderError(provider, response); + if (response.status !== 400 && response.status !== 422) { + return baseError; + } + + try { + const body = (await response.text()).trim(); + if (!body) { + return baseError; + } + const parsed = JSON.parse(body) as { + error?: { message?: unknown }; + detail?: unknown; + }; + const detail = + (typeof parsed.error?.message === 'string' && parsed.error.message) || + (typeof parsed.detail === 'string' && parsed.detail) || + body; + return `${baseError} ${detail.slice(0, 300)}`; + } catch { + return baseError; + } +} + +async function resolveLocalProviderConnection( + provider: LocalTaskModelProviderId, + input?: LocalProviderConnectionInput, +): Promise { + const envNames = LOCAL_PROVIDER_CONNECTION_ENV[provider]; + const persisted = await getPersistedEnvironmentVariableValues([ + envNames.baseUrl, + ...(envNames.apiKey ? [envNames.apiKey] : []), + ]); + const baseUrl = + input?.baseUrl?.trim() || + process.env[envNames.baseUrl]?.trim() || + persisted[envNames.baseUrl]?.trim(); + + if (!baseUrl) { + return null; + } + + try { + const url = new URL(baseUrl); + if (!['http:', 'https:'].includes(url.protocol)) { + return null; + } + } catch { + return null; + } + + return { + baseUrl, + apiKey: + input?.apiKey?.trim() || + (envNames.apiKey ? process.env[envNames.apiKey] : null) || + (envNames.apiKey ? persisted[envNames.apiKey] : null) || + null, + }; +} + +function buildLocalProviderHeaders(connection: LocalProviderConnection) { + return connection.apiKey + ? { Authorization: `Bearer ${connection.apiKey}` } + : undefined; +} + +function getNumber(value: unknown) { + return typeof value === 'number' && Number.isFinite(value) && value > 0 + ? value + : null; +} + +function buildLocalProviderModel( + provider: LocalTaskModelProviderId, + slug: string, + displayName?: string, + info?: Record, +): TaskModelLookupResult { + const metadataPatch = { + contextWindow: + getNumber(info?.max_input_tokens) ?? + getNumber(info?.max_tokens) ?? + getNumber(info?.context_window), + inputPricePerToken: getNumber(info?.input_cost_per_token), + outputPricePerToken: getNumber(info?.output_cost_per_token), + }; + const hasMetadata = Object.values(metadataPatch).some( + (value) => value !== null, + ); + const model = buildTaskModelOption({ + id: `${provider}/${slug}`, + displayName: displayName?.trim() || slug, + metadata: hasMetadata ? mergeMetadata(null, metadataPatch) : null, + }); + + return { + modelId: model.id, + displayName: model.displayName, + family: model.family, + metadata: model.metadata ?? null, + }; +} + +async function fetchLocalProviderModels( + provider: LocalTaskModelProviderId, + connection: LocalProviderConnection, +): Promise { + const paths = + provider === 'ollama' ? ['/api/tags', '/v1/models'] : ['/v1/models']; + let lastError: string | null = null; + + for (const path of paths) { + try { + const response = await fetch( + getLocalProviderBaseUrl(connection.baseUrl, path), + { + headers: buildLocalProviderHeaders(connection), + signal: AbortSignal.timeout(LOCAL_PROVIDER_REQUEST_TIMEOUT_MS), + }, + ); + if (!response.ok) { + lastError = getLocalProviderError(provider, response); + continue; + } + + const payload = (await response.json()) as LocalProviderModelResponse; + let litellmModelInfo = payload.model_info; + if (provider === 'litellm') { + try { + const infoResponse = await fetch( + getLocalProviderBaseUrl(connection.baseUrl, '/model/info'), + { + headers: buildLocalProviderHeaders(connection), + signal: AbortSignal.timeout(LOCAL_PROVIDER_REQUEST_TIMEOUT_MS), + }, + ); + if (infoResponse.ok) { + const infoPayload = + (await infoResponse.json()) as LocalProviderModelResponse; + const infoEntries = Object.fromEntries( + (infoPayload.data ?? []).flatMap((entry) => { + const modelName = entry.model_name ?? entry.id; + return modelName && entry.model_info + ? [[modelName, entry.model_info]] + : []; + }), + ); + litellmModelInfo = { + ...litellmModelInfo, + ...infoPayload.model_info, + ...infoEntries, + }; + } + } catch { + // LiteLLM's metadata endpoint is optional; model discovery still works. + } + } + const entries = path === '/api/tags' ? payload.models : payload.data; + const models = (entries ?? []) + .map((entry) => { + const slug = + entry.id?.trim() || + entry.name?.trim() || + entry.model?.trim() || + entry.model_name?.trim(); + return slug + ? buildLocalProviderModel( + provider, + slug, + entry.model_name ?? entry.name, + entry.model_info ?? litellmModelInfo?.[slug], + ) + : null; + }) + .filter((model): model is TaskModelLookupResult => model !== null) + .sort((left, right) => left.modelId.localeCompare(right.modelId)); + + return { + models, + modelCount: models.length, + recommendedModels: getRecommendedLocalProviderModels(models), + error: null, + }; + } catch { + lastError = getLocalProviderNetworkError(provider); + } + } + + return { + models: [], + modelCount: 0, + recommendedModels: [], + error: lastError ?? getLocalProviderNetworkError(provider), + }; +} + +export async function discoverProviderModelsCommand( + auth: UserAuthSuccess, + input: { provider: LocalTaskModelProviderId } & LocalProviderConnectionInput, +): Promise { + assertAdmin(auth); + const connection = await resolveLocalProviderConnection( + input.provider, + input, + ); + + if (!connection) { + return { + models: [], + modelCount: 0, + recommendedModels: [], + error: 'Save a valid endpoint URL before discovering models.', + }; + } + + return fetchLocalProviderModels(input.provider, connection); +} + +export async function qualifyProviderModelCommand( + auth: UserAuthSuccess, + input: { + provider: LocalTaskModelProviderId; + modelId: string; + } & LocalProviderConnectionInput, +): Promise<{ success: true } | { success: false; error: string }> { + assertAdmin(auth); + const connection = await resolveLocalProviderConnection( + input.provider, + input, + ); + const modelId = input.modelId.trim(); + + if (!connection) { + return { + success: false, + error: 'Save a valid endpoint URL before qualifying a model.', + }; + } + if (!modelId) { + return { success: false, error: 'Choose a model before qualifying it.' }; + } + + try { + const response = await fetch( + getLocalProviderBaseUrl(connection.baseUrl, '/v1/chat/completions'), + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...buildLocalProviderHeaders(connection), + }, + body: JSON.stringify({ + model: modelId.replace(`${input.provider}/`, ''), + messages: [{ role: 'user', content: 'Reply with pong.' }], + stream: true, + tool_choice: { + type: 'function', + function: { name: 'ping' }, + }, + tools: [ + { + type: 'function', + function: { + name: 'ping', + description: 'Returns a short health-check response.', + parameters: { type: 'object', properties: {} }, + }, + }, + ], + }), + signal: AbortSignal.timeout(LOCAL_PROVIDER_REQUEST_TIMEOUT_MS), + }, + ); + if (!response.ok) { + return { + success: false, + error: await getQualificationError(input.provider, response), + }; + } + + if (!response.headers.get('content-type')?.includes('text/event-stream')) { + return { + success: false, + error: + 'The provider returned a non-streaming response. Check OpenAI-compatible streaming support.', + }; + } + + const reader = response.body?.getReader(); + if (!reader) { + return { + success: false, + error: + 'The provider accepted the request but did not return a streaming response body.', + }; + } + + const decoder = new TextDecoder(); + let streamText = ''; + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) { + break; + } + streamText += decoder.decode(chunk.value, { stream: true }); + } + } finally { + reader.releaseLock(); + } + + if ( + !streamText.includes('"tool_calls"') || + !streamText.includes('"ping"') + ) { + return { + success: false, + error: + 'The model streamed a response but did not call the required tool. Choose a model with OpenAI-compatible tool calling.', + }; + } + return { success: true }; + } catch { + return { + success: false, + error: getLocalProviderNetworkError(input.provider), + }; + } +} + /** * Providers without a single-model lookup endpoint (Vercel AI Gateway and * direct labs such as Anthropic or OpenAI) resolve display name and pricing @@ -1155,6 +1736,27 @@ export async function lookupTaskModelCommand( }; } + const localProvider = LOCAL_TASK_MODEL_PROVIDER_IDS.find((provider) => + modelId.startsWith(`${provider}/`), + ); + if (localProvider) { + const discovery = await discoverProviderModelsCommand(auth, { + provider: localProvider, + }); + const discoveredModel = discovery.models.find( + (model) => model.modelId === modelId, + ); + + return ( + discoveredModel ?? { + modelId, + displayName: null, + family: null, + metadata: null, + } + ); + } + // Only the OpenRouter model API supports single-model lookup; every other // provider (Vercel AI Gateway and direct labs such as Anthropic) resolves // from the models.dev catalog. diff --git a/apps/web/src/trpc/routers/_app.ts b/apps/web/src/trpc/routers/_app.ts index 61cfce496..5f7401f1c 100644 --- a/apps/web/src/trpc/routers/_app.ts +++ b/apps/web/src/trpc/routers/_app.ts @@ -294,10 +294,12 @@ import { } from '../commands/access-policy'; import { deleteTaskModelProviderCommand, + discoverProviderModelsCommand, getLaunchTaskModelsCommand, getTaskModelProviderSetupCommand, getTaskModelSettingsCommand, lookupTaskModelCommand, + qualifyProviderModelCommand, refreshTaskModelMetadataCommand, saveTaskModelProviderCommand, suggestTaskModelsCommand, @@ -1657,6 +1659,7 @@ export const appRouter = createRouter({ provider: z.enum(SETUP_MODEL_PROVIDER_IDS), apiKey: z.string().trim().optional(), additionalEnvValues: z.record(z.string().trim()).optional(), + modelId: z.string().trim().optional(), }), ) .mutation(({ ctx: { auth }, input }) => @@ -1673,6 +1676,31 @@ export const appRouter = createRouter({ deleteTaskModelProviderCommand(auth, input), ), + discoverProviderModels: protectedProcedure + .input( + z.object({ + provider: z.enum(['ollama', 'vllm', 'litellm']), + baseUrl: z.string().trim().optional(), + apiKey: z.string().trim().optional(), + }), + ) + .mutation(({ ctx: { auth }, input }) => + discoverProviderModelsCommand(auth, input), + ), + + qualifyProviderModel: protectedProcedure + .input( + z.object({ + provider: z.enum(['ollama', 'vllm', 'litellm']), + modelId: z.string().trim().min(1), + baseUrl: z.string().trim().optional(), + apiKey: z.string().trim().optional(), + }), + ) + .mutation(({ ctx: { auth }, input }) => + qualifyProviderModelCommand(auth, input), + ), + lookup: protectedProcedure .input( z.object({ @@ -1865,6 +1893,7 @@ export const appRouter = createRouter({ provider: z.enum(SETUP_MODEL_PROVIDER_IDS), apiKey: z.string().trim().optional(), additionalEnvValues: z.record(z.string().trim()).optional(), + modelId: z.string().trim().optional(), }), ) .mutation(({ ctx: { auth }, input }) => diff --git a/apps/worker/src/run-task/agent-home.test.ts b/apps/worker/src/run-task/agent-home.test.ts index 1265d48fa..c07eb7a45 100644 --- a/apps/worker/src/run-task/agent-home.test.ts +++ b/apps/worker/src/run-task/agent-home.test.ts @@ -278,6 +278,71 @@ describe('generateOpenCodeConfig provider support', () => { expect(result.configContent).not.toContain('ROOMOTE_CLOUD_TOKEN'); }); + it('configures selected OpenAI-compatible providers with direct-mode fallbacks', () => { + const result = generateOpenCodeConfig({ + homeDir: createHomeDir(), + runtimeEnv: { + R_MODEL: 'ollama/qwen3-coder', + R_SMALL_MODEL: 'vllm/meta-llama/Llama-3.3-70B-Instruct', + R_VISION_MODEL: 'litellm/gpt-4.1-mini', + VLLM_BASE_URL: 'https://vllm.example.com/v1', + VLLM_API_KEY: 'vllm-key', + LITELLM_BASE_URL: 'https://litellm.example.com/v1', + LITELLM_API_KEY: 'litellm-key', + }, + }); + const config = JSON.parse(result.configContent) as { + provider: Record }>; + }; + + expect(config.provider.ollama).toMatchObject({ + npm: '@ai-sdk/openai-compatible', + options: { baseURL: 'http://127.0.0.1:11434/v1' }, + models: { 'qwen3-coder': { name: 'qwen3-coder' } }, + }); + expect(config.provider.ollama?.options.apiKey).toBe('ollama'); + expect(config.provider.vllm).toMatchObject({ + options: { + baseURL: 'https://vllm.example.com/v1', + apiKey: '{env:VLLM_API_KEY}', + }, + }); + expect(config.provider.litellm).toMatchObject({ + options: { + baseURL: 'https://litellm.example.com/v1', + apiKey: '{env:LITELLM_API_KEY}', + }, + }); + }); + + it('rebases gateway-backed compatible providers without an Ollama key', () => { + const result = generateOpenCodeConfig({ + homeDir: createHomeDir(), + runtimeEnv: { + R_MODEL: 'ollama/qwen3-coder', + R_SMALL_MODEL: 'vllm/meta-llama/Llama-3.3-70B-Instruct', + R_VISION_MODEL: 'litellm/gpt-4.1-mini', + R_INFERENCE_GATEWAY_URL: 'https://api.example.com/api/inference/', + }, + }); + const config = JSON.parse(result.configContent) as { + provider: Record }>; + }; + + expect(config.provider.ollama?.options).toMatchObject({ + baseURL: 'https://api.example.com/api/inference/ollama/v1', + apiKey: '{env:ROOMOTE_CLOUD_TOKEN}', + }); + expect(config.provider.vllm?.options).toMatchObject({ + baseURL: 'https://api.example.com/api/inference/vllm/v1', + apiKey: '{env:ROOMOTE_CLOUD_TOKEN}', + }); + expect(config.provider.litellm?.options).toMatchObject({ + baseURL: 'https://api.example.com/api/inference/litellm/v1', + apiKey: '{env:ROOMOTE_CLOUD_TOKEN}', + }); + }); + it('strips disabled-provider credentials and removes a stale Vertex file', () => { const homeDir = createHomeDir(); const credentialsPath = join( diff --git a/apps/worker/src/run-task/agent-home.ts b/apps/worker/src/run-task/agent-home.ts index f4789fa3b..91e4d89db 100644 --- a/apps/worker/src/run-task/agent-home.ts +++ b/apps/worker/src/run-task/agent-home.ts @@ -68,6 +68,33 @@ export const OPENCODE_AUTH_FILE_NAME = 'auth.json'; const OPENROUTER_PROVIDER_ID = 'openrouter'; +const OPENAI_COMPATIBLE_PROVIDER_CONFIGS = { + ollama: { + name: 'Ollama', + baseUrlEnvVarName: 'OLLAMA_BASE_URL', + fallbackBaseUrl: 'http://127.0.0.1:11434/v1', + apiKeyEnvVarName: undefined, + keyless: true, + }, + vllm: { + name: 'vLLM', + baseUrlEnvVarName: 'VLLM_BASE_URL', + fallbackBaseUrl: 'http://127.0.0.1:8000/v1', + apiKeyEnvVarName: 'VLLM_API_KEY', + keyless: false, + }, + litellm: { + name: 'LiteLLM', + baseUrlEnvVarName: 'LITELLM_BASE_URL', + fallbackBaseUrl: 'http://127.0.0.1:4000/v1', + apiKeyEnvVarName: 'LITELLM_API_KEY', + keyless: false, + }, +} as const; + +type OpenAiCompatibleProviderId = + keyof typeof OPENAI_COMPATIBLE_PROVIDER_CONFIGS; + /** * OpenRouter identifies the calling application through the `HTTP-Referer` * and `X-Title` request headers rather than the standard `User-Agent`. @@ -644,6 +671,82 @@ function mergeBedrockMantleProviderConfig( }; } +function mergeOpenAiCompatibleProviderConfig( + providerConfig: Record, + runtimeEnv: Record, + modelIds: Array, +): Record { + let merged = providerConfig; + + for (const providerId of Object.keys( + OPENAI_COMPATIBLE_PROVIDER_CONFIGS, + ) as OpenAiCompatibleProviderId[]) { + const provider = OPENAI_COMPATIBLE_PROVIDER_CONFIGS[providerId]; + const prefix = `${providerId}/`; + const modelIdsForProvider = [ + ...new Set( + modelIds.flatMap((modelId) => { + const normalized = modelId?.trim(); + + return normalized?.startsWith(prefix) + ? [normalized.slice(prefix.length)] + : []; + }), + ), + ]; + + if (modelIdsForProvider.length === 0) { + continue; + } + + const existingProvider = asRecord(merged[providerId]); + const existingOptions = asRecord(existingProvider.options); + const existingModels = asRecord(existingProvider.models); + const directApiKey = provider.apiKeyEnvVarName + ? runtimeEnv[provider.apiKeyEnvVarName]?.trim() + : undefined; + + merged = { + ...merged, + [providerId]: { + ...existingProvider, + npm: '@ai-sdk/openai-compatible', + name: provider.name, + options: { + ...existingOptions, + baseURL: + runtimeEnv[provider.baseUrlEnvVarName]?.trim() || + (!provider.keyless ? runtimeEnv.OPENAI_BASE_URL?.trim() : '') || + provider.fallbackBaseUrl, + // OpenAI-compatible clients require an API key even though Ollama + // itself accepts unauthenticated requests. + ...(directApiKey + ? { apiKey: `{env:${provider.apiKeyEnvVarName}}` } + : provider.keyless + ? { apiKey: 'ollama' } + : runtimeEnv.OPENAI_API_KEY?.trim() + ? { apiKey: '{env:OPENAI_API_KEY}' } + : {}), + }, + models: { + ...existingModels, + ...Object.fromEntries( + modelIdsForProvider.map((modelId) => [ + modelId, + { + name: modelId, + ...asRecord(existingModels[modelId]), + }, + ]), + ), + }, + }, + }; + } + + return merged; +} + /** * When the dequeue env carries an inference gateway URL, rebase each * gateway-covered provider that a selected model uses onto the gateway. The @@ -654,6 +757,7 @@ function mergeBedrockMantleProviderConfig( function mergeInferenceGatewayProviderConfig( providerConfig: Record, runtimeEnv: Record, + modelIds: Array, ): Record { const gatewayUrl = runtimeEnv[INFERENCE_GATEWAY_URL_ENV_VAR_NAME]?.trim(); const servedKeyNames = parseInferenceGatewayKeys( @@ -662,10 +766,7 @@ function mergeInferenceGatewayProviderConfig( const routeChatGptThroughGateway = runtimeEnv[INFERENCE_GATEWAY_CHATGPT_ENV_VAR_NAME] === '1'; - if ( - !gatewayUrl || - (servedKeyNames.length === 0 && !routeChatGptThroughGateway) - ) { + if (!gatewayUrl) { return providerConfig; } @@ -720,6 +821,27 @@ function mergeInferenceGatewayProviderConfig( } } + // These providers are configured explicitly because OpenCode has no catalog + // entries for their arbitrary model IDs. Rebase only registered gateway + // providers: vLLM stays direct until the gateway declares its route. + for (const providerId of Object.keys( + OPENAI_COMPATIBLE_PROVIDER_CONFIGS, + ) as OpenAiCompatibleProviderId[]) { + const gatewayProvider = getInferenceGatewayProvider(providerId); + + if ( + gatewayProvider && + modelIds.some((modelId) => modelId?.trim().startsWith(`${providerId}/`)) + ) { + merged = rebaseProviderOntoGateway( + merged, + providerId, + gatewayUrl, + gatewayProvider, + ); + } + } + return merged; } @@ -1292,14 +1414,19 @@ function resolveModelBackedOpenCodeConfig( ]; const providerConfig = mergeInferenceGatewayProviderConfig( mergeBedrockMantleProviderConfig( - mergeOpenRouterVariantAliasModels( - providerReasoningConfig, - variantAliases, + mergeOpenAiCompatibleProviderConfig( + mergeOpenRouterVariantAliasModels( + providerReasoningConfig, + variantAliases, + ), + runtimeEnv, + configuredModelIds, ), runtimeEnv, configuredModelIds, ), runtimeEnv, + configuredModelIds, ); return { diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index d661d144a..cc41a9379 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -1311,7 +1311,7 @@ export const llmUsageEvents = pgTable( .default(0), costSource: text('cost_source') .notNull() - .$type<'opencode_message' | 'missing'>(), + .$type<'opencode_message' | 'litellm_gateway' | 'missing'>(), pricingMetadata: jsonb('pricing_metadata') .notNull() .default({}) diff --git a/packages/sdk/src/server/lib/task-runs/__tests__/record-task-inference-usage.test.ts b/packages/sdk/src/server/lib/task-runs/__tests__/record-task-inference-usage.test.ts index ee68c7cc7..1ec37055c 100644 --- a/packages/sdk/src/server/lib/task-runs/__tests__/record-task-inference-usage.test.ts +++ b/packages/sdk/src/server/lib/task-runs/__tests__/record-task-inference-usage.test.ts @@ -196,6 +196,28 @@ describe('recordTaskInferenceUsage', () => { }); }); + it('accepts LiteLLM response costs', async () => { + await recordLlmUsage({ + source: 'litellm', + usageType: 'inference', + eventKey: 'litellm-response-cost-1', + providerId: 'litellm', + modelId: 'gpt-4o', + costMicroUsd: 123, + costSource: 'litellm_gateway', + }); + + const [event] = await db + .select() + .from(llmUsageEvents) + .where(eq(llmUsageEvents.eventKey, 'litellm-response-cost-1')); + + expect(event).toMatchObject({ + costMicroUsd: 123, + costSource: 'litellm_gateway', + }); + }); + it('upserts non-task usage by event key and keeps model rows separate', async () => { await recordLlmUsage({ source: 'router', diff --git a/packages/sdk/src/server/lib/task-runs/record-task-inference-usage.ts b/packages/sdk/src/server/lib/task-runs/record-task-inference-usage.ts index 3387979c1..fb9896936 100644 --- a/packages/sdk/src/server/lib/task-runs/record-task-inference-usage.ts +++ b/packages/sdk/src/server/lib/task-runs/record-task-inference-usage.ts @@ -1,6 +1,7 @@ import { db, eq, llmUsageEvents, taskRuns } from '@roomote/db/server'; +import type { LlmUsageCostSource } from '@roomote/types'; -type TaskInferenceUsageCostSource = 'opencode_message' | 'missing'; +type TaskInferenceUsageCostSource = LlmUsageCostSource; interface RecordTaskInferenceUsageInput { runId: number; @@ -183,7 +184,12 @@ export async function recordLlmUsage( totalTokens, contextTokens, costMicroUsd: clampOptionalCostMicroUsd(input.costMicroUsd), - costSource, + // The database column is a text column; its generated type can lag this + // forward-compatible usage contract. + costSource: costSource as + | 'opencode_message' + | 'litellm_gateway' + | 'missing', messageCreatedAt: input.messageCreatedAt ?? null, messageCompletedAt: input.messageCompletedAt ?? null, pricingMetadata: input.pricingMetadata ? { ...input.pricingMetadata } : {}, diff --git a/packages/sdk/src/server/routers/llm-usage.ts b/packages/sdk/src/server/routers/llm-usage.ts index 557d51435..9b069e27d 100644 --- a/packages/sdk/src/server/routers/llm-usage.ts +++ b/packages/sdk/src/server/routers/llm-usage.ts @@ -1,6 +1,7 @@ import { z } from 'zod'; import { and, db, eq, or, taskRuns, tasks } from '@roomote/db/server'; +import { LLM_USAGE_COST_SOURCES } from '@roomote/types'; import { router, userOnlyProcedure } from '../trpc'; import { recordLlmUsage } from '../lib/task-runs/record-task-inference-usage'; @@ -32,10 +33,7 @@ export const llmUsageRouter = router({ totalTokens: nonNegativeNumber, contextTokens: nonNegativeNumber, costMicroUsd: z.number().nonnegative().nullable().optional(), - costSource: z - .enum(['opencode_message', 'missing']) - .nullable() - .optional(), + costSource: z.enum(LLM_USAGE_COST_SOURCES).nullable().optional(), messageCreatedAt: z.date().nullable().optional(), messageCompletedAt: z.date().nullable().optional(), pricingMetadata: z.record(z.unknown()).nullable().optional(), diff --git a/packages/sdk/src/server/routers/task-runs.ts b/packages/sdk/src/server/routers/task-runs.ts index 7f97cd6a1..2d2295f25 100644 --- a/packages/sdk/src/server/routers/task-runs.ts +++ b/packages/sdk/src/server/routers/task-runs.ts @@ -28,6 +28,7 @@ import { snapshotResumeSchema, sourceControlProviderSchema, ROOMOTE_RUNTIME_TASK_MESSAGE_PROTOCOL, + LLM_USAGE_COST_SOURCES, type AcpPersistedEnvelope, } from '@roomote/types'; import { @@ -430,7 +431,7 @@ export const taskRunsRouter = router({ totalTokens: z.number().int().nonnegative().nullable().optional(), contextTokens: z.number().int().nonnegative().nullable().optional(), costMicroUsd: z.number().int().nonnegative().nullable().optional(), - costSource: z.enum(['opencode_message', 'missing']).nullable().optional(), + costSource: z.enum(LLM_USAGE_COST_SOURCES).nullable().optional(), messageCreatedAt: z.date().nullable().optional(), messageCompletedAt: z.date().nullable().optional(), details: z.record(z.unknown()).nullable().optional(), diff --git a/packages/types/src/__tests__/inference-gateway.test.ts b/packages/types/src/__tests__/inference-gateway.test.ts index 534f98bf8..1768312e9 100644 --- a/packages/types/src/__tests__/inference-gateway.test.ts +++ b/packages/types/src/__tests__/inference-gateway.test.ts @@ -57,7 +57,9 @@ describe('inference gateway key lookups', () => { it('maps each covered env var name back to its provider', () => { for (const envVarName of INFERENCE_GATEWAY_PROVIDER_ENV_VAR_NAMES) { const provider = getInferenceGatewayProviderByEnvVarName(envVarName); - expect(provider?.envVarNames).toContain(envVarName); + expect(provider?.gatewayEnvVarNames ?? provider?.envVarNames).toContain( + envVarName, + ); expect(isInferenceGatewayCoveredEnvVar(envVarName)).toBe(true); } }); @@ -79,6 +81,21 @@ describe('inference gateway key lookups', () => { expect(isInferenceGatewayCoveredEnvVar('SOME_OTHER_KEY')).toBe(false); }); + it('registers local OpenAI-compatible endpoint providers', () => { + expect(getInferenceGatewayProvider('litellm')).toMatchObject({ + upstreamBaseUrlEnvVarName: 'LITELLM_BASE_URL', + gatewayEnvVarNames: ['LITELLM_BASE_URL', 'LITELLM_API_KEY'], + }); + expect(getInferenceGatewayProvider('ollama')).toMatchObject({ + upstreamBaseUrlEnvVarName: 'OLLAMA_BASE_URL', + gatewayEnvVarNames: ['OLLAMA_BASE_URL'], + }); + expect(getInferenceGatewayProvider('vllm')).toMatchObject({ + upstreamBaseUrlEnvVarName: 'VLLM_BASE_URL', + gatewayEnvVarNames: ['VLLM_BASE_URL', 'VLLM_API_KEY'], + }); + }); + it('parses a comma-separated served-keys value', () => { expect( parseInferenceGatewayKeys('ANTHROPIC_API_KEY, OPENROUTER_API_KEY'), diff --git a/packages/types/src/__tests__/opencode-reasoning.test.ts b/packages/types/src/__tests__/opencode-reasoning.test.ts index 07e0cffe5..1212a41d5 100644 --- a/packages/types/src/__tests__/opencode-reasoning.test.ts +++ b/packages/types/src/__tests__/opencode-reasoning.test.ts @@ -119,6 +119,12 @@ describe('buildOpenCodeModelReasoningOptions', () => { ).toEqual({ reasoningEffort: 'medium' }); }); + it('clamps LiteLLM xhigh reasoning to high', () => { + expect( + buildOpenCodeModelReasoningOptions('litellm/openai/gpt-5.4', 'xhigh'), + ).toEqual({ reasoningEffort: 'high' }); + }); + it('returns null for malformed model ids', () => { expect( buildOpenCodeModelReasoningOptions('no-provider', 'high'), diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index dcc3e550f..964aae5d3 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -30,6 +30,7 @@ export * from './linear'; export * from './llm-citation-artifacts'; export * from './live-previews'; export * from './logging'; +export * from './llm-usage'; export * from './inference-gateway'; export * from './model-provider-config'; export * from './recommended-task-models'; diff --git a/packages/types/src/inference-gateway.ts b/packages/types/src/inference-gateway.ts index 943417cfb..f59e1be4d 100644 --- a/packages/types/src/inference-gateway.ts +++ b/packages/types/src/inference-gateway.ts @@ -85,13 +85,21 @@ export interface InferenceGatewayProvider { * in precedence order. Empty for non-key auth strategies. */ envVarNames: readonly string[]; + /** + * Deployment variables withheld from sandboxes when this provider is served + * through the gateway. Endpoint providers include their base URL here even + * when they have no API key, so the endpoint topology stays server-side. + */ + gatewayEnvVarNames?: readonly string[]; /** How the gateway authenticates upstream. Defaults to `api-key`. */ authStrategy?: InferenceGatewayAuthStrategy; /** * Upstream API base. May contain a `{region}` placeholder resolved * per-request from `region` below. */ - upstreamBaseUrl: string; + upstreamBaseUrl?: string; + /** Deployment env var holding an operator-configured upstream base URL. */ + upstreamBaseUrlEnvVarName?: string; /** * When set, every allowed request path is rewritten to this fixed upstream * path (the ChatGPT Codex backend collapses `/responses` and @@ -104,7 +112,9 @@ export interface InferenceGatewayProvider { */ region?: { envVarName: string; default: string }; /** How the upstream expects its API key when the gateway forwards. */ - authHeader: InferenceGatewayAuthHeader; + authHeader?: InferenceGatewayAuthHeader; + /** A configured upstream key is forwarded when present but is not required. */ + optionalApiKey?: boolean; /** * Upstream inference endpoints the gateway forwards, matched exactly. * Everything else is rejected so a run token can only reach inference @@ -294,6 +304,41 @@ export const INFERENCE_GATEWAY_PROVIDERS: readonly InferenceGatewayProvider[] = allowedPaths: ANTHROPIC_COMPATIBLE_INFERENCE_PATHS, openCodeBaseUrlSuffix: '/v1', }, + { + id: 'litellm', + name: 'LiteLLM', + envVarNames: ['LITELLM_API_KEY'], + gatewayEnvVarNames: ['LITELLM_BASE_URL', 'LITELLM_API_KEY'], + upstreamBaseUrlEnvVarName: 'LITELLM_BASE_URL', + authHeader: { name: 'authorization', scheme: 'bearer' }, + allowedPaths: OPENAI_COMPATIBLE_INFERENCE_PATHS, + openCodeBaseUrlSuffix: '/v1', + }, + { + id: 'ollama', + name: 'Ollama', + envVarNames: [], + gatewayEnvVarNames: ['OLLAMA_BASE_URL'], + upstreamBaseUrlEnvVarName: 'OLLAMA_BASE_URL', + optionalApiKey: true, + allowedPaths: [ + ...OPENAI_COMPATIBLE_INFERENCE_PATHS, + '/api/tags', + '/api/ps', + ], + openCodeBaseUrlSuffix: '/v1', + }, + { + id: 'vllm', + name: 'vLLM', + envVarNames: ['VLLM_API_KEY'], + gatewayEnvVarNames: ['VLLM_BASE_URL', 'VLLM_API_KEY'], + upstreamBaseUrlEnvVarName: 'VLLM_BASE_URL', + authHeader: { name: 'authorization', scheme: 'bearer' }, + optionalApiKey: true, + allowedPaths: OPENAI_COMPATIBLE_INFERENCE_PATHS, + openCodeBaseUrlSuffix: '/v1', + }, { // ChatGPT subscription: the gateway holds the OAuth record, mints a // fresh access token per request, adds the account-id header, and @@ -324,7 +369,9 @@ export const INFERENCE_GATEWAY_PROVIDERS: readonly InferenceGatewayProvider[] = * env vars. */ export const INFERENCE_GATEWAY_PROVIDER_ENV_VAR_NAMES: readonly string[] = - INFERENCE_GATEWAY_PROVIDERS.flatMap((provider) => provider.envVarNames); + INFERENCE_GATEWAY_PROVIDERS.flatMap( + (provider) => provider.gatewayEnvVarNames ?? provider.envVarNames, + ); export function getInferenceGatewayProvider( providerId: string, @@ -352,7 +399,7 @@ export function getInferenceGatewayProviderByEnvVarName( envVarName: string, ): InferenceGatewayProvider | undefined { return INFERENCE_GATEWAY_PROVIDERS.find((provider) => - provider.envVarNames.includes(envVarName), + (provider.gatewayEnvVarNames ?? provider.envVarNames).includes(envVarName), ); } diff --git a/packages/types/src/llm-usage.ts b/packages/types/src/llm-usage.ts new file mode 100644 index 000000000..05e4ef759 --- /dev/null +++ b/packages/types/src/llm-usage.ts @@ -0,0 +1,7 @@ +export const LLM_USAGE_COST_SOURCES = [ + 'opencode_message', + 'litellm_gateway', + 'missing', +] as const; + +export type LlmUsageCostSource = (typeof LLM_USAGE_COST_SOURCES)[number]; diff --git a/packages/types/src/model-provider-config.test.ts b/packages/types/src/model-provider-config.test.ts index 94a32a1f3..6d89ac3e8 100644 --- a/packages/types/src/model-provider-config.test.ts +++ b/packages/types/src/model-provider-config.test.ts @@ -229,6 +229,9 @@ describe('SETUP_MODEL_PROVIDER_CATALOG', () => { 'amazon-bedrock', 'google', 'xai', + 'litellm', + 'ollama', + 'vllm', 'chatgpt', ], ); @@ -236,6 +239,11 @@ describe('SETUP_MODEL_PROVIDER_CATALOG', () => { it('keeps recommended-model slugs and default models under each provider prefix', () => { for (const provider of SETUP_MODEL_PROVIDER_CATALOG) { + if ('dynamicModels' in provider && provider.dynamicModels) { + expect(provider.defaultRoomoteModel).toBe(''); + expect(provider.suggestedTaskModels).toEqual([]); + continue; + } // ChatGPT serves openai/ models; the Bedrock setup surface serves the // worker's custom bedrock-mantle/ OpenCode provider. const expectedPrefix = @@ -262,6 +270,10 @@ describe('SETUP_MODEL_PROVIDER_CATALOG', () => { SETUP_MODEL_PROVIDER_CATALOG; for (const provider of providers) { + if (provider.dynamicModels) { + continue; + } + const expectedPrefix = provider.id === 'chatgpt' ? 'openai/' diff --git a/packages/types/src/model-provider-config.ts b/packages/types/src/model-provider-config.ts index 6014f6c91..d5804ca8c 100644 --- a/packages/types/src/model-provider-config.ts +++ b/packages/types/src/model-provider-config.ts @@ -40,7 +40,7 @@ export type SetupModelProviderId = (typeof SETUP_MODEL_PROVIDER_IDS)[number]; * var (`envVarName`); OAuth providers are connected through a dedicated flow * and carry no env var. */ -export type SetupModelProviderAuthKind = 'api-key' | 'oauth'; +export type SetupModelProviderAuthKind = 'api-key' | 'endpoint' | 'oauth'; /** * The default-model roles a deployment configures: one model per kind of @@ -124,6 +124,8 @@ export type SetupModelProviderDescriptor = { defaultRoomoteModel: string; authKind: SetupModelProviderAuthKind; suggestedTaskModels: readonly SuggestedTaskModel[]; + /** Models are discovered from the configured endpoint, not a static catalog. */ + dynamicModels?: boolean; /** * Hides the provider from the setup wizard and settings connect surfaces * unless it is already connected (saved or runtime env). The catalog entry @@ -449,6 +451,52 @@ export const SETUP_MODEL_PROVIDER_CATALOG = [ 'grok-4-5': 'xai/grok-4.5', }), }, + { + id: 'litellm', + label: 'LiteLLM', + envVarName: 'LITELLM_BASE_URL', + envVarLabel: 'Endpoint URL', + additionalEnvFields: [ + { + envVarName: 'LITELLM_API_KEY', + label: 'API key', + secret: true, + required: true, + }, + ], + defaultRoomoteModel: '', + authKind: 'endpoint', + suggestedTaskModels: [], + dynamicModels: true, + }, + { + id: 'ollama', + label: 'Ollama', + envVarName: 'OLLAMA_BASE_URL', + envVarLabel: 'Endpoint URL', + defaultRoomoteModel: '', + authKind: 'endpoint', + suggestedTaskModels: [], + dynamicModels: true, + }, + { + id: 'vllm', + label: 'vLLM', + envVarName: 'VLLM_BASE_URL', + envVarLabel: 'Endpoint URL', + additionalEnvFields: [ + { + envVarName: 'VLLM_API_KEY', + label: 'API key', + secret: true, + required: false, + }, + ], + defaultRoomoteModel: '', + authKind: 'endpoint', + suggestedTaskModels: [], + dynamicModels: true, + }, { id: CHATGPT_SUBSCRIPTION_PROVIDER_ID, label: 'ChatGPT (subscription)', @@ -1153,7 +1201,17 @@ export function buildSetupModelStatus(input: { isConfiguredEnvValue(runtimeEnv[name]); const isPersisted = (name: string) => persistedEnvVarNameSet.has(name); const additionalEnvValues = Object.fromEntries( - getSetupModelProviderAdditionalEnvFields(provider) + [ + ...(provider.authKind === 'endpoint' && provider.envVarName + ? [ + { + envVarName: provider.envVarName, + secret: false, + }, + ] + : []), + ...getSetupModelProviderAdditionalEnvFields(provider), + ] .filter((field) => !field.secret) .map((field) => { const runtimeValue = normalizeOptionalString( diff --git a/packages/types/src/opencode-reasoning.ts b/packages/types/src/opencode-reasoning.ts index ef046b1ec..8d2c4909c 100644 --- a/packages/types/src/opencode-reasoning.ts +++ b/packages/types/src/opencode-reasoning.ts @@ -173,6 +173,12 @@ export function buildOpenCodeModelReasoningOptions( case 'anthropic': case 'bedrock-mantle': return buildAnthropicReasoningOptions(selection.modelID, reasoningEffort); + case 'litellm': + // LiteLLM's OpenAI-compatible endpoint rejects `xhigh`; keep the + // highest accepted setting instead of failing the inference request. + return { + reasoningEffort: reasoningEffort === 'xhigh' ? 'high' : reasoningEffort, + }; default: return { reasoningEffort }; } diff --git a/packages/types/src/task-models.ts b/packages/types/src/task-models.ts index ea5ce6ffe..9adba55f9 100644 --- a/packages/types/src/task-models.ts +++ b/packages/types/src/task-models.ts @@ -28,6 +28,9 @@ export const ENABLED_DIRECT_TASK_MODEL_PROVIDER_IDS = [ 'amazon-bedrock', 'google', 'xai', + 'litellm', + 'ollama', + 'vllm', ] as const; /**