From c520496cc7eff20524e944a1e2556f1096b069ea Mon Sep 17 00:00:00 2001 From: Spikel Date: Fri, 7 Aug 2026 11:46:36 +0800 Subject: [PATCH 1/2] Rewrite BitRouter vs LiteLLM around claims that hold up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LiteLLM now ships a Rust core ("Rust core with Python SDK") and publishes benchmarks at ~0.05ms added overhead. Our page claimed they were a Python gateway at ~20ms+ against our ~5ms p50 — a comparison we lose on their own numbers, on a page whose whole job is credibility. Reframed the argument onto ground that survives their December roadmap: - Lead with all-in-one vs. focused-and-extensible around the act → observe → evaluate → learn loop. It's the one difference they can't ship past. - Promote self-serve cloud vs. quote-only Enterprise (verified against litellm.ai/pricing). - Demote Rust to "Rust-accelerated vs. Rust-native": their recommended mode is a PyO3 bridge in the Python wheel, the standalone server is beta with fewer routes, and their own endgame keeps Python in a sidecar for plugins and SSO. Argue dependency surface, not language speed. - Concede properly: LiteLLM's in-process library and its embeddings/rerank/ audio coverage are real advantages we don't match. Both now appear in the table and the migration guide, not just the prose. The security paragraph describes the 2026 RCE by incident class rather than CVE number, and explicitly disclaims immunity — our MCP gateway executes tools too, and Rust prevents none of that bug class. Also: - Trim the compare table 11 rows → 5; drop the routing-overhead row. - Correct platform fee to 0% markup on the OpenRouter and Portkey tables. - Rewrite the LiteLLM terminal animations, which asserted ~85ms p99 / GIL and "LiteLLM: none of the above" (false — they ship MCP). No numbers. - Wire two terminals into the page and whitelist CompareTerminal, which was registered in mdx-components but missing from the docs lint whitelist. - migrate-from-litellm claimed LiteLLM was "self-hosted only". It isn't. Numbers left as TK- placeholders pending real benchmarks. Co-Authored-By: Claude Opus 5 --- .../landing/compare/compare-programs.tsx | 49 +++++++++---------- .../(guide)/overview/bitrouter-vs-litellm.mdx | 40 ++++++++++++--- content/docs/guides/migrate-from-litellm.mdx | 20 +++++--- lib/docs-sync/constants.mjs | 1 + 4 files changed, 67 insertions(+), 43 deletions(-) diff --git a/components/landing/compare/compare-programs.tsx b/components/landing/compare/compare-programs.tsx index 36c21d3..d3a08ea 100644 --- a/components/landing/compare/compare-programs.tsx +++ b/components/landing/compare/compare-programs.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { Ok, Err, Dim, Faint } from "../zed/terminal"; +import { Ok, Err, Warn, Dim, Faint } from "../zed/terminal"; export type TermStep = { term: string; prog: () => unknown[] }; export type CompareRow = { feat: string; them: string; br: string }; @@ -21,20 +21,21 @@ export const COMPARE_REGISTRY: Record = { term: "deploy · bitrouter", prog: () => [ ["print", LiteLLM production, 240], - ["print", ✗ postgres · redis · docker-compose · nginx, 200], + ["print", ⚠ python runtime · web framework · datastore, 200], ["print", BitRouter, 300], - ["print", ✓ bitrouter serve · 1 binary · 0 deps, 320], - ["print", ● ready in 340ms, 600], + ["print", ✓ bitrouter serve · 1 static binary · 0 deps, 320], + ["print", {" ↓ no interpreter in the image"}, 600], ["loop", 2000], ], }, "02": { - term: "latency · 1k req/s", + term: "runtime · where the rust stops", prog: () => [ - ["print", tail latency · 1k concurrent req/s, 240], - ["print", LiteLLM ~85ms p99 Python GIL, 240], - ["print", BitRouter ~12ms p99 Rust async, 320], - ["print", {" ↓ no GIL · no thread contention"}, 600], + ["print", LiteLLM · rust core for translation, 240], + ["print", ⚠ python owns auth · routing · callbacks, 200], + ["print", ⚠ standalone rust server beta · fewer routes, 320], + ["print", BitRouter · rust end to end, 240], + ["print", ✓ one binary · nothing in a sidecar, 600], ["loop", 2000], ], }, @@ -43,31 +44,25 @@ export const COMPARE_REGISTRY: Record = { prog: () => [ ["print", configuring policy default, 320], ["print", ✓ MCP gateway active, 180], + ["print", ✓ ACP gateway active, 180], ["print", ✓ KYA identity active, 180], - ["print", ✓ injection detect active, 180], ["print", ✓ x402 payments active, 420], - ["print", LiteLLM: none of the above, 600], + ["print", LiteLLM: MCP only, 600], ["loop", 2000], ], }, }, rows: [ - { feat: "Open source", them: "✓ MIT", br: "✓ Apache 2.0" }, - { feat: "Single binary (no dependencies)", them: "✗ Postgres + Redis + Docker", br: "✓" }, - { feat: "Agent gateway (MCP / ACP / Skills)", them: "✗", br: "✓ built-in" }, - { feat: "Autonomous agent payments (x402)", them: "✗", br: "✓" }, - { feat: "KYA agent identity", them: "✗", br: "✓" }, - { feat: "Prompt injection detection", them: "✗", br: "✓" }, - { feat: "Multi-provider failover mid-run", them: "✗ manual config", br: "✓ automatic" }, - { feat: "Routing overhead", them: "✗ ~20ms+ (asyncio overhead)", br: "✓ ~5ms p50" }, - { feat: "Per-run cost attribution", them: "✗", br: "✓" }, - { feat: "BYOK support", them: "✓", br: "✓" }, - { feat: "Platform fee (hosted option)", them: "— self-host only", br: "✓ 2% stablecoin / 5% card" }, + { feat: "Provider & endpoint coverage", them: "✓ 100+ providers, embeddings / rerank / audio", br: "⚠ SOTA-tier chat, extend by PR" }, + { feat: "What you deploy", them: "⚠ Python runtime + web framework + database", br: "✓ one static binary, no deps" }, + { feat: "Agent gateway (MCP / ACP / Skills)", them: "⚠ MCP only", br: "✓ built-in" }, + { feat: "Routing that learns from outcomes", them: "✗ static config", br: "✓ act → observe → evaluate → learn" }, + { feat: "Hosted option", them: "⚠ Enterprise, quote-only", br: "✓ self-serve, no sales call" }, ], tradeoffs: [ - "You want a Python-native library embedded directly in your application code with SDK-level call hooks", - "Your stack is pure Python and you need framework callbacks (async generators, middleware)", - "You're using LiteLLM's extensive provider mapping for non-standard model endpoints", + "You need breadth on day one — 100+ providers plus embeddings, rerank, and audio endpoints", + "You want the library, not the proxy: in-process Python calls with SDK-level hooks and middleware", + "You need enterprise checkboxes shipped today — SSO/SAML, audit logs, RBAC, compliance certifications", ], }, @@ -119,7 +114,7 @@ export const COMPARE_REGISTRY: Record = { { feat: "Routing overhead", them: "✗ ~30ms", br: "✓ ~5ms p50" }, { feat: "Per-run cost attribution", them: "✗", br: "✓" }, { feat: "BYOK support", them: "✓", br: "✓" }, - { feat: "Platform fee", them: "✗ 5.5% card only", br: "✓ 2% stablecoin / 5% card" }, + { feat: "Platform fee", them: "✗ 5.5% card only", br: "✓ 0% markup" }, ], tradeoffs: [ "You need the widest possible model catalog and don't want to operate any infrastructure", @@ -179,7 +174,7 @@ export const COMPARE_REGISTRY: Record = { { feat: "Routing overhead", them: "⚠ ~15ms p50", br: "✓ ~5ms p50" }, { feat: "Per-run cost attribution", them: "✓", br: "✓" }, { feat: "BYOK support", them: "✓", br: "✓" }, - { feat: "Platform fee", them: "— varies by plan", br: "✓ 2% stablecoin / 5% card" }, + { feat: "Platform fee", them: "— varies by plan", br: "✓ 0% markup" }, ], tradeoffs: [ "Your team already relies on Portkey's prompt management, versioning, and caching workflows", diff --git a/content/docs/(guide)/overview/bitrouter-vs-litellm.mdx b/content/docs/(guide)/overview/bitrouter-vs-litellm.mdx index e0fc505..5afaa88 100644 --- a/content/docs/(guide)/overview/bitrouter-vs-litellm.mdx +++ b/content/docs/(guide)/overview/bitrouter-vs-litellm.mdx @@ -1,23 +1,47 @@ --- title: BitRouter vs LiteLLM -description: LiteLLM is a Python-native library that lives inside your app; BitRouter is a single Rust binary with zero dependencies and ~7× better tail latency under load. An honest side-by-side. +description: LiteLLM is the all-in-one AI gateway with the widest provider coverage in the category. BitRouter is a focused, extensible one built around a loop that learns. An honest side-by-side. --- -LiteLLM is the popular MIT-licensed, Python-native option — and if you want a library that lives *inside* your application with SDK-level call hooks, that's exactly its strength. The differences show up in operations and latency. +LiteLLM is the default choice for good reason. It's MIT-licensed, it speaks to 100+ providers, and whatever gateway feature you're looking for, it probably already has a config key for it. We're not going to pretend otherwise. If you want a Python library that lives *inside* your process — `from litellm import completion`, no proxy at all — BitRouter has no answer for that, and it isn't close. -## Operations and latency +So the useful question isn't which is better. It's whether you want a box with everything in it, or a small thing that gets better at its job. -A production LiteLLM deployment typically means Postgres, Redis, and Docker Compose to run the proxy, and its Python runtime carries the GIL: under concurrent load, tail latency climbs to **~85ms p99**. BitRouter is a single binary with zero dependencies (`bitrouter serve`, ready in ~340ms) and a Rust async core that holds **~12ms p99** at 1k req/s. +## All-in-one proxy vs. a router that learns -## What each one routes +LiteLLM's bet is coverage: every provider, every gateway feature, in one proxy you configure. It works. Ours is the opposite, because we don't think the interesting problem is feature count — it's that routers don't learn. Every gateway on the market forwards your thousandth call exactly the way it forwarded your first. BitRouter wraps your agent loop in a second one — [act → observe → evaluate → learn](/docs/overview/what-is-bitrouter) — and folds what it observes back into the routing policy. Cheaper paths that still reached the goal get promoted; ones that didn't get escalated back. -On agent features the gap is structural: an MCP/ACP gateway, KYA identity, injection detection, and autonomous x402 payments are built into BitRouter and absent from LiteLLM. LiteLLM routes model calls; BitRouter routes *agent runs* — and wraps them in an [act → observe → evaluate → learn](/docs/overview/what-is-bitrouter) loop that tightens routing every run, instead of forwarding to a static route. +"Minimal" here means minimal to *operate*, not minimal in ambition. An MCP gateway, an ACP gateway for sub-agents, KYA identity, and x402 payments are all in the box — but they serve that one loop, and it's still one binary and one `bitrouter.yaml`. Anything we don't ship, you add: the [provider registry](/docs/gateway-and-routing/provider-selection) takes a YAML file and a PR. + + -## LiteLLM is the right call when +## Self-serve cloud vs. enterprise-only + +LiteLLM's hosted offering is an enterprise product: quote-only, sized to your request volume, priced through a sales conversation. SSO, audit logs, RBAC, and support SLAs live behind it. If you're a two-person team, the only real option is self-hosting it yourself. + +[BitRouter Cloud](/docs/overview/quickstart) is self-serve for anyone with a credit card — no call, no minimum, no seat count, and **0% markup** on what providers charge. The Core stays Apache 2.0 and free to self-host forever; bring your own keys or run a local model and you owe us nothing. + +## Rust-accelerated vs. Rust-native + +A correction to the record first: LiteLLM is no longer a Python-performance story. They've moved their translation hot path to a Rust core and publish benchmarks showing roughly **TK-LITELLM-RUST-OVERHEAD** of added overhead. That's a real result, and we'd rather say so than pretend it isn't happening. + + + +What remains is where the Rust stops. In LiteLLM's recommended mode it's a bridge inside the Python wheel — Python still owns auth, routing, callbacks, and spend tracking, and you still deploy a Python runtime, a web framework, and a database. The standalone Rust server is beta and covers fewer routes. Their roadmap puts the full transition at the end of 2026, and even that endgame runs Python in a sidecar for plugins, guardrails, and SSO by design. BitRouter is one statically linked binary: `bitrouter serve`, ready in **TK-BR-STARTUP**, **TK-BR-MEMORY** resident, no interpreter in the image. + +That gap matters less for latency than you'd think, and more for what you inherit. A Python proxy inherits its framework's vulnerabilities — 2026 saw a critical unauthenticated RCE in LiteLLM built by chaining a command-injection bug in its MCP endpoints with a host-header bypass in an upstream Python web framework. Both were disclosed and patched properly, and we're not claiming immunity: our MCP gateway executes tools too, and no language prevents a design bug. We're claiming a smaller blast radius — no interpreter loading code at runtime, no transitive framework chain to inherit a bypass from, and a lot less of it to audit. + +## Use LiteLLM when + +**You need breadth on day one.** LiteLLM covers 100+ providers plus embeddings, rerank, audio, and image endpoints out of the box. BitRouter deliberately ships deep support for SOTA-tier chat models and extends by PR — if you need Bedrock's fourteenth variant or a rerank endpoint this afternoon, that's LiteLLM. + +**You want the library, not the proxy.** LiteLLM's real superpower is being importable — in-process calls with SDK-level hooks, async generators, and framework middleware. If your stack is pure Python and you want the gateway inside your app rather than next to it, use LiteLLM. + +**You need enterprise checkboxes today.** SSO/SAML, audit logs, RBAC, compliance certifications, support SLAs — LiteLLM Enterprise has them shipped and we're still building ours. If you like where we're going and have enterprise requirements, [talk to us](https://cal.com/bitrouter/founder-call) — that roadmap is very much open to being shaped. -You want a Python-native library embedded directly in your app code, your stack is pure Python and you rely on framework callbacks (async generators, middleware), or you lean on LiteLLM's extensive provider mapping for non-standard model endpoints. If you'd rather move the routing out of your process, the [migration guide](/docs/guides/migrate-from-litellm) covers it. +Already running LiteLLM and want to try the other shape? The [migration guide](/docs/guides/migrate-from-litellm) keeps your provider mappings. ## Next steps diff --git a/content/docs/guides/migrate-from-litellm.mdx b/content/docs/guides/migrate-from-litellm.mdx index 45e103e..583698c 100644 --- a/content/docs/guides/migrate-from-litellm.mdx +++ b/content/docs/guides/migrate-from-litellm.mdx @@ -11,12 +11,12 @@ LiteLLM is a Python SDK and self-hosted proxy for unifying access to 100+ LLM pr | | LiteLLM Proxy | BitRouter | |---|---|---| -| **Runtime** | Python | Rust (single static binary) | -| **Production deps** | Postgres + Redis + Docker/K8s | None | -| **Deployment modes** | Self-hosted only | Local binary **or** hosted (`api.bitrouter.ai`) — same OpenAI-compatible endpoint | +| **Runtime** | Python host with a Rust core for translation | Rust (single static binary) | +| **Production deps** | Python runtime + web framework + Postgres/Redis | None | +| **Deployment modes** | Self-hosted, or Enterprise cloud (quote-only) | Local binary **or** self-serve hosted (`api.bitrouter.ai`) — same OpenAI-compatible endpoint | +| **Provider & endpoint coverage** | 100+ providers, plus embeddings / rerank / audio | SOTA-tier chat models; extend by PR | | **Agentic auth & payment** | None | x402 / MPP autonomous payment (cloud mode) | -| **Design focus** | All-in-one LLM gateway: admin UI, virtual keys, budgets, with agent gateways added alongside | Agent-first proxy: MCP / ACP / Skills, agent firewall, agentic payment as the core surface | -| **Agent protocol surface** | MCP, A2A, Skills, CLI — bolted onto the horizontal gateway | MCP, ACP, Skills, CLI — the product, not an add-on | +| **Design focus** | All-in-one LLM gateway: admin UI, virtual keys, budgets, with agent gateways alongside | Agent-first proxy: MCP / ACP / Skills, agent firewall, agentic payment as the core surface | | **License** | MIT (SDK) / paid enterprise tier | Apache 2.0 throughout | ## Two things worth highlighting @@ -29,7 +29,7 @@ This matters when your agent should *pay per request* without you provisioning k ### 2. Agent-native, not all-in-one -LiteLLM has shipped MCP, A2A, Skills, and a CLI alongside its horizontal LLM gateway — virtual keys, team budgets, spend dashboards, admin UI all included. BitRouter inverts that: agent primitives *are* the product, the team-admin stack is intentionally minimal. What you get on the BitRouter surface: +LiteLLM has shipped MCP, A2A, Skills, and a CLI alongside its horizontal LLM gateway — virtual keys, team budgets, spend dashboards, and an admin UI all included. BitRouter inverts the emphasis: agent primitives *are* the product, and the team-admin stack is intentionally minimal. What you get on the BitRouter surface: - [MCP gateway](/docs/gateway-and-routing/mcp-gateway) — proxy MCP servers so agents discover tools across hosts. - [ACP gateway](/docs/gateway-and-routing/acp-gateway) — first-class support for the Agent Client Protocol used by Claude Code, Codex, OpenCode, and others. @@ -82,7 +82,7 @@ Fallbacks and provider selection that you'd configure with `litellm.Router` move ### From the LiteLLM Proxy -The proxy migration replaces the Python process + Postgres + Redis with a single binary. Install and launch: +The proxy migration replaces the Python host and its datastores with a single binary. Install and launch: @@ -129,7 +129,11 @@ To skip the local proxy entirely, point clients at `https://api.bitrouter.ai/v1` ## What BitRouter intentionally doesn't ship -To set expectations honestly: BitRouter does not ship a built-in admin UI for team/user budgets, virtual-key generation by API, or a spend analytics dashboard at parity with LiteLLM Enterprise. Per-workspace key scoping in cloud mode and env-var-scoped keys in local mode cover the common cases, but if your migration depends on per-user virtual keys with quotas enforced inside the proxy, plan for that gap or stay on LiteLLM for those workloads. +To set expectations honestly, two gaps are worth checking before you start. + +**Non-chat endpoints.** LiteLLM covers embeddings, rerank, audio, and image endpoints across 100+ providers out of the box. BitRouter ships deep support for SOTA-tier chat models and extends by PR to the [provider registry](/docs/gateway-and-routing/provider-selection). If your workload leans on embeddings or rerank today, either add them to the registry first or keep those calls on LiteLLM. + +**Team administration.** BitRouter does not ship a built-in admin UI for team/user budgets, virtual-key generation by API, or a spend analytics dashboard at parity with LiteLLM Enterprise. Per-workspace key scoping in cloud mode and env-var-scoped keys in local mode cover the common cases, but if your migration depends on per-user virtual keys with quotas enforced inside the proxy, plan for that gap or stay on LiteLLM for those workloads. ## Migration checklist diff --git a/lib/docs-sync/constants.mjs b/lib/docs-sync/constants.mjs index 0c5a8b6..cab74d8 100644 --- a/lib/docs-sync/constants.mjs +++ b/lib/docs-sync/constants.mjs @@ -10,6 +10,7 @@ export const COMPONENT_WHITELIST = [ "ProvidersTable", "CalInline", "CompareTable", + "CompareTerminal", ]; // Where synced authored docs land (will be gitignored at the Plan B cutover). From dee8be24eb0ce20964c9990ebf7c7fb7e7b0294a Mon Sep 17 00:00:00 2001 From: Spikel Date: Fri, 7 Aug 2026 11:50:36 +0800 Subject: [PATCH 2/2] Replace TK placeholders with qualitative claims; drop terminals from page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page can't ship with TK- markers rendering literally, so the three numbers are now qualitative: - LiteLLM's overhead is "sub-millisecond" — their published 0.05ms, stated as their result. Conceding it plainly is stronger than hedging, since the paragraph's point is that the distinction is where the Rust stops, not speed. - BitRouter "cold-starting in well under a second and holding tens of megabytes resident" — conservative bands, still worth replacing with measurements before this is load-bearing anywhere. Also removes both blocks from the page. The rewritten terminal programs stay in compare-programs.tsx (correct now, just unused again), as does the CompareTerminal whitelist entry — the component is registered in mdx-components.tsx, so the whitelist was out of sync regardless of whether a page uses it today. Co-Authored-By: Claude Opus 5 --- content/docs/(guide)/overview/bitrouter-vs-litellm.mdx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/content/docs/(guide)/overview/bitrouter-vs-litellm.mdx b/content/docs/(guide)/overview/bitrouter-vs-litellm.mdx index 5afaa88..644d74f 100644 --- a/content/docs/(guide)/overview/bitrouter-vs-litellm.mdx +++ b/content/docs/(guide)/overview/bitrouter-vs-litellm.mdx @@ -13,8 +13,6 @@ LiteLLM's bet is coverage: every provider, every gateway feature, in one proxy y "Minimal" here means minimal to *operate*, not minimal in ambition. An MCP gateway, an ACP gateway for sub-agents, KYA identity, and x402 payments are all in the box — but they serve that one loop, and it's still one binary and one `bitrouter.yaml`. Anything we don't ship, you add: the [provider registry](/docs/gateway-and-routing/provider-selection) takes a YAML file and a PR. - - ## Self-serve cloud vs. enterprise-only @@ -25,11 +23,9 @@ LiteLLM's hosted offering is an enterprise product: quote-only, sized to your re ## Rust-accelerated vs. Rust-native -A correction to the record first: LiteLLM is no longer a Python-performance story. They've moved their translation hot path to a Rust core and publish benchmarks showing roughly **TK-LITELLM-RUST-OVERHEAD** of added overhead. That's a real result, and we'd rather say so than pretend it isn't happening. - - +A correction to the record first: LiteLLM is no longer a Python-performance story. They've moved their translation hot path to a Rust core and publish benchmarks showing **sub-millisecond** added overhead. That's a real result, and we'd rather say so than pretend it isn't happening. -What remains is where the Rust stops. In LiteLLM's recommended mode it's a bridge inside the Python wheel — Python still owns auth, routing, callbacks, and spend tracking, and you still deploy a Python runtime, a web framework, and a database. The standalone Rust server is beta and covers fewer routes. Their roadmap puts the full transition at the end of 2026, and even that endgame runs Python in a sidecar for plugins, guardrails, and SSO by design. BitRouter is one statically linked binary: `bitrouter serve`, ready in **TK-BR-STARTUP**, **TK-BR-MEMORY** resident, no interpreter in the image. +What remains is where the Rust stops. In LiteLLM's recommended mode it's a bridge inside the Python wheel — Python still owns auth, routing, callbacks, and spend tracking, and you still deploy a Python runtime, a web framework, and a database. The standalone Rust server is beta and covers fewer routes. Their roadmap puts the full transition at the end of 2026, and even that endgame runs Python in a sidecar for plugins, guardrails, and SSO by design. BitRouter is one statically linked binary: `bitrouter serve`, cold-starting in well under a second and holding tens of megabytes resident, with no interpreter and no web framework in the image. That gap matters less for latency than you'd think, and more for what you inherit. A Python proxy inherits its framework's vulnerabilities — 2026 saw a critical unauthenticated RCE in LiteLLM built by chaining a command-injection bug in its MCP endpoints with a host-header bypass in an upstream Python web framework. Both were disclosed and patched properly, and we're not claiming immunity: our MCP gateway executes tools too, and no language prevents a design bug. We're claiming a smaller blast radius — no interpreter loading code at runtime, no transitive framework chain to inherit a bypass from, and a lot less of it to audit.