[Feat] Add self-hosted inference providers

apps/api/src/handlers/inference/index.ts

@@ -2,6 +2,7 @@ import { Hono } from 'hono';
 
 import { formatSingleLineLog } from '@roomote/types';
 import { db, eq, taskRuns } from '@roomote/db/server';
+import { recordLlmUsage } from '@roomote/sdk/server';
 
 import type { Variables } from '../../types';
 import { fetchWithLongLivedStreamDispatcher } from '../long-lived-fetch';
@@ -49,6 +50,59 @@ const REQUEST_HEADER_DENYLIST = new Set([
   'x-real-ip',
 ]);
 
+function recordLiteLlmResponseCost(options: {
+  requestId: string;
+  runId: number;
+  headers: Headers;
+}): void {
+  const cost = Number(options.headers.get('x-litellm-response-cost'));
+
+  if (!Number.isFinite(cost) || cost <= 0) {
+    return;
+  }
+
+  const modelGroup = options.headers.get('x-litellm-model-group')?.trim();
+
+  void db.query.taskRuns
+    .findFirst({
+      where: eq(taskRuns.id, options.runId),
+      columns: { taskId: true },
+    })
+    .then((run) => {
+      if (!run?.taskId) {
+        return undefined;
+      }
+
+      return recordLlmUsage({
+        eventKey: `inference-gateway:${options.requestId}`,
+        source: 'inference-gateway',
+        usageType: 'inference',
+        taskId: run.taskId,
+        runId: options.runId,
+        providerId: 'litellm',
+        modelId: modelGroup ? `litellm/${modelGroup}` : null,
+        inputTokens: null,
+        outputTokens: null,
+        reasoningTokens: null,
+        cacheReadTokens: null,
+        cacheWriteTokens: null,
+        totalTokens: null,
+        contextTokens: null,
+        costMicroUsd: Math.round(cost * 1_000_000),
+        costSource: 'litellm_gateway',
+      });
+    })
+    .catch((error) => {
+      console.warn(
+        formatSingleLineLog('Failed to record LiteLLM response cost', {
+          requestId: options.requestId,
+          runId: options.runId,
+          error: error instanceof Error ? error.message : String(error),
+        }),
+      );
+    });
+}
+
 function buildUpstreamRequestHeaders(
   requestHeaders: Headers,
   injectedHeaders: Record<string, string>,
@@ -188,6 +242,14 @@ inference.on(['POST', 'GET'], '/:provider/*', async (c) => {
       },
     );
 
+    if (providerId === 'litellm') {
+      recordLiteLlmResponseCost({
+        requestId,
+        runId: auth.runId,
+        headers: upstreamResponse.headers,
+      });
+    }
+
     if (!upstreamResponse.ok) {
       console.warn(
         formatSingleLineLog(`${logPrefix} Upstream returned non-OK status`, {

apps/api/src/handlers/inference/registry.ts

@@ -46,7 +46,7 @@ export async function resolveGatewayUpstream(
     resolveProviderUpstreamBaseUrl(provider),
   ]);
 
-  if (!apiKey) {
+  if (!apiKey && !provider.optionalApiKey) {
     return {
       ok: false,
       status: 404,
@@ -58,12 +58,15 @@ export async function resolveGatewayUpstream(
     ok: true,
     resolved: {
       upstreamUrl: `${upstreamBaseUrl}${upstreamPath}${search}`,
-      headers: {
-        [provider.authHeader.name]: formatProviderAuthHeaderValue(
-          provider,
-          apiKey,
-        ),
-      },
+      headers:
+        apiKey && provider.authHeader
+          ? {
+              [provider.authHeader.name]: formatProviderAuthHeaderValue(
+                provider,
+                apiKey,
+              ),
+            }
+          : {},
     },
   };
 }
@@ -81,9 +84,9 @@ async function resolveChatGptUpstream(
     };
   }
 
-  const upstreamUrl = `${provider.upstreamBaseUrl}${provider.collapseToPath ?? ''}`;
+  const upstreamUrl = `${provider.upstreamBaseUrl!}${provider.collapseToPath ?? ''}`;
   const headers: Record<string, string> = {
-    [provider.authHeader.name]: formatProviderAuthHeaderValue(
+    [provider.authHeader!.name]: formatProviderAuthHeaderValue(
       provider,
       token.access,
     ),
@@ -150,8 +153,25 @@ function hasTraversalOrEncodedSlash(upstreamPath: string): boolean {
 async function resolveProviderUpstreamBaseUrl(
   provider: InferenceGatewayProvider,
 ): Promise<string> {
+  if (provider.upstreamBaseUrlEnvVarName) {
+    const configuredBaseUrl = await resolveModelProviderEnvValue([
+      provider.upstreamBaseUrlEnvVarName,
+    ]);
+
+    if (!configuredBaseUrl) {
+      throw new Error(
+        `${provider.upstreamBaseUrlEnvVarName} must be configured for ${provider.name}.`,
+      );
+    }
+
+    return validateDynamicUpstreamBaseUrl(
+      configuredBaseUrl,
+      provider.upstreamBaseUrlEnvVarName,
+    );
+  }
+
   if (!provider.region) {
-    return provider.upstreamBaseUrl;
+    return provider.upstreamBaseUrl!;
   }
 
   const region =
@@ -164,12 +184,56 @@ async function resolveProviderUpstreamBaseUrl(
     );
   }
 
-  return provider.upstreamBaseUrl.replace('{region}', region);
+  return provider.upstreamBaseUrl!.replace('{region}', region);
+}
+
+function validateDynamicUpstreamBaseUrl(
+  value: string,
+  envVarName: string,
+): string {
+  let url: URL;
+
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`${envVarName} must be an absolute HTTP(S) URL.`);
+  }
+
+  if (
+    (url.protocol !== 'http:' && url.protocol !== 'https:') ||
+    url.username ||
+    url.password ||
+    url.search ||
+    url.hash
+  ) {
+    throw new Error(
+      `${envVarName} must be an HTTP(S) URL without credentials, query parameters, or fragments.`,
+    );
+  }
+
+  const normalizedPath = stripDynamicEndpointVersionSuffix(url.pathname);
+  url.pathname = normalizedPath;
+
+  return url.toString().replace(/\/+$/u, '');
+}
+
+function stripDynamicEndpointVersionSuffix(pathname: string): string {
+  let end = pathname.length;
+
+  while (end > 0 && pathname.charCodeAt(end - 1) === 47 /* '/' */) {
+    end -= 1;
+  }
+
+  const withoutTrailingSlashes = pathname.slice(0, end);
+
+  return withoutTrailingSlashes.endsWith('/v1')
+    ? withoutTrailingSlashes.slice(0, -3) || '/'
+    : withoutTrailingSlashes || '/';
 }
 
 function formatProviderAuthHeaderValue(
   provider: InferenceGatewayProvider,
   apiKey: string,
 ): string {
-  return provider.authHeader.scheme === 'bearer' ? `Bearer ${apiKey}` : apiKey;
+  return provider.authHeader?.scheme === 'bearer' ? `Bearer ${apiKey}` : apiKey;
 }

apps/docs/docs.json

@@ -43,7 +43,12 @@
           {
             "group": "Models and Inference",
             "root": "models",
-            "pages": []
+            "expanded": true,
+            "pages": [
+              "providers/inference/litellm",
+              "providers/inference/ollama",
+              "providers/inference/vllm"
+            ]
           },
           {
             "group": "Communications",

apps/docs/environment-variables.mdx

@@ -188,6 +188,11 @@ as per-task auth tokens or workspace paths.
 | `GOOGLE_GENERATIVE_AI_API_KEY`         | Provider key | Alternate Google/Gemini provider key forwarded when configured or inferred.                                                             |
 | `AWS_BEARER_TOKEN_BEDROCK`             | Provider key | Amazon Bedrock Mantle API key. Can also be saved from **Settings > Models**.                                                            |
 | `AWS_REGION`                           | Provider key | AWS region where the Bedrock API key was created. Defaults to `us-east-1` at runtime when unset.                                        |
+| `LITELLM_BASE_URL`                     | LiteLLM      | LiteLLM endpoint URL, usually including its `/v1` path.                                                                                 |
+| `LITELLM_API_KEY`                      | LiteLLM      | LiteLLM gateway API key. Required when configuring LiteLLM.                                                                              |
+| `OLLAMA_BASE_URL`                      | Ollama       | Ollama endpoint URL. Use the service root, for example `http://ollama:11434`.                                                           |
+| `VLLM_BASE_URL`                        | vLLM         | vLLM OpenAI-compatible endpoint URL, usually including its `/v1` path.                                                                  |
+| `VLLM_API_KEY`                         | Optional     | Bearer API key for a vLLM endpoint that requires authentication.                                                                        |
 
 ### Sandbox providers