From 3ecfb5f052d198643b6f7cd8064987cc118069b7 Mon Sep 17 00:00:00 2001 From: Kushagra Kanaujia Date: Mon, 16 Nov 2026 22:42:05 -0800 Subject: [PATCH 1/2] Add throttle fit: the fewest GPUs your model needs, offline estimate Memory math per GPU type (weights plus full-context KV for every chat at once, tensor parallel limits, KV copies past the KV head count), monthly cost at your prices, a --current check for spare cards, and the autotune command that measures the shortlist. Labelled ESTIMATE; never predicts speed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 + src/throttle/cli.py | 17 +- .../console/docs/guides/fewest-gpus.md | 47 ++ src/throttle/console/server.py | 1 + src/throttle/fit.py | 454 ++++++++++++++++++ tests/test_cli.py | 1 + tests/test_fit.py | 141 ++++++ 7 files changed, 673 insertions(+), 1 deletion(-) create mode 100644 src/throttle/console/docs/guides/fewest-gpus.md create mode 100644 src/throttle/fit.py create mode 100644 tests/test_fit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a5305..d907440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to Throttle will be documented in this file. ## [Unreleased] +### Added +- `throttle fit`: the fewest GPUs your model needs at your context length + and number of chats at once, per GPU type, before you rent anything. + Offline, no GPU, no keys. Memory math only (weights plus the KV cache for + every chat at the full context, with vLLM/SGLang's tensor parallel limits + and the KV copies when cards outnumber KV heads), labelled ESTIMATE, and it + never predicts speed. `--price NAME=$/HOUR` adds the monthly cost of each + option; `--current h100=4` says if fewer cards look like enough and what + that would save if a measurement agrees. It ends with the `throttle + autotune` (or, for llama.cpp, `throttle check`) command that measures the + shortlist. 11 built-in models, 12 built-in GPUs, `--config config.json` + for any other model, `--gpu name=GB` for any other card, `--json`. + ## [0.8.0] - 2026-11-12 ### Added diff --git a/src/throttle/cli.py b/src/throttle/cli.py index 537a798..ff387d7 100644 --- a/src/throttle/cli.py +++ b/src/throttle/cli.py @@ -133,7 +133,8 @@ never moved silently. Serving tools (advanced), for the server your own model runs on: - check ($/M tokens with 95% CIs), autotune, coldstart, compare, gate, + fit (fewest GPUs for your model, offline estimate), check ($/M tokens with + 95% CIs), autotune, coldstart, compare, gate, savings, watch, ui, proxy, sessions. Evidence runs: plan, smoke, benchmark, golden, golden-report. @@ -657,6 +658,17 @@ def build_parser() -> argparse.ArgumentParser: ) add_coldstart_arguments(coldstart) + from .fit import FIT_DESCRIPTION, FIT_EPILOG, add_fit_arguments + fit = subparsers.add_parser( + "fit", + help="the fewest GPUs your model needs at your context and users (offline " + "estimate, no GPU); then the command to measure it", + description=FIT_DESCRIPTION, + epilog=FIT_EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + add_fit_arguments(fit) + from .gate import GATE_DESCRIPTION, add_gate_arguments gate = subparsers.add_parser( "gate", @@ -2488,6 +2500,9 @@ def main(argv: Sequence[str] | None = None) -> int: if args.command == "autotune": from .autotune import handle_autotune return handle_autotune(args) + if args.command == "fit": + from .fit import handle_fit + return handle_fit(args) if args.command == "coldstart": from .coldstart import handle_coldstart return handle_coldstart(args) diff --git a/src/throttle/console/docs/guides/fewest-gpus.md b/src/throttle/console/docs/guides/fewest-gpus.md new file mode 100644 index 0000000..eeb2fb2 --- /dev/null +++ b/src/throttle/console/docs/guides/fewest-gpus.md @@ -0,0 +1,47 @@ +# Fewest GPUs for your model + +*Before you rent cards: how many does your model need at your traffic?* + +`throttle fit` is memory math. It runs on your laptop in a second, with no GPU and no keys, and every number it prints is an **estimate**. + +```bash +throttle fit --model llama-3.3-70b --quant fp8 --context 32k --users 16 \ + --gpu h100 --gpu l40s --price h100=2.49 --price l40s=0.99 --current h100=4 +``` + +For each GPU it shows: + +| Column | Meaning | +|---|---| +| fewest cards | the smallest card count that holds the weights plus a full-length KV cache for every chat at once | +| chats they hold | how many full-length chats that many cards can hold | +| per month | that many cards at your `--price`, 730 hours a month | + +With `--current`, it tells you if fewer cards look like enough, and what that would save **if a measurement agrees**. + +### What it does not know + +- **Speed.** More cards can be slower, because they have to talk to each other. Measure before you change anything. +- **Your real chat lengths.** It assumes every chat at the full `--context` at the same time. Real traffic is usually lighter, so this errs on the safe side. +- **Exotic attention.** Sliding-window and latent attention models need less KV memory than this math assumes. + +### Then measure it + +`throttle fit` ends with the command to run on your server, for example: + +```bash +throttle autotune --url http://localhost:8000 --model meta-llama/Llama-3.3-70B-Instruct \ + --gpu-hourly-rate 2.49 \ + --launch "vllm serve {model} --tensor-parallel-size {tp}" --space tp=2,4 +``` + +That gives you real cost per million tokens for each card count, with confidence intervals. + +### Any model, any card + +```bash +throttle fit --config ./config.json --params 32.8 --quant awq --context 8k --users 8 +throttle fit --model qwen2.5-32b --quant q4_k_m --engine llamacpp --gpu rtx3090 --gpu 1060=6 +``` + +`--list-models` and `--list-gpus` show what is built in. `--json` prints the result for scripts. diff --git a/src/throttle/console/server.py b/src/throttle/console/server.py index 30e6371..31b1f53 100644 --- a/src/throttle/console/server.py +++ b/src/throttle/console/server.py @@ -77,6 +77,7 @@ ("guides/sglang", "SGLang"), ("guides/ollama", "Ollama"), ("guides/other-engines", "Other servers"), + ("guides/fewest-gpus", "Fewest GPUs for your model"), ("guides/compare-gpus", "Compare two GPUs"), ("guides/test-quantization", "Test a quantization"), ("guides/ci-github-action", "Gate costs in CI"), diff --git a/src/throttle/fit.py b/src/throttle/fit.py new file mode 100644 index 0000000..afa836d --- /dev/null +++ b/src/throttle/fit.py @@ -0,0 +1,454 @@ +"""``throttle fit``: the fewest GPUs your model needs, before you rent any. + +Give a model, the context length your users need and how many of them talk to +it at once. Throttle works out, for each GPU type, the fewest cards that hold +the weights plus the KV cache for that many full-length chats, how many such +chats that many cards can hold, and (if you give prices) what each option +costs a month. + +It is memory math only: offline, no GPU, no keys, nothing sent anywhere. +Every number is an ESTIMATE. It does not predict speed: more cards can be +slower, not faster, because they have to talk to each other. So it ends with +the command that measures the shortlist on your real server. + +Pass ``--current GPU=COUNT`` (for example ``--current h100=4``) and it tells +you if fewer cards look like enough, which is where the money usually is. +""" + +from __future__ import annotations + +import argparse +import json +import math +import shlex +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +from . import style as style_module + +EXIT_OK = 0 +EXIT_DOES_NOT_FIT = 1 +EXIT_USAGE = 2 + +HOURS_PER_MONTH = 730 +GIB = 1024 ** 3 +# What vLLM lets itself use by default (--gpu-memory-utilization 0.9), and a +# flat allowance per card for the CUDA context, activations and graphs. +DEFAULT_MEMORY_FRACTION = 0.90 +DEFAULT_OVERHEAD_GB = 1.5 +MAX_GPUS = 8 + +FIT_DESCRIPTION = __doc__.split("\n\n", 1)[1] if __doc__ else "" +FIT_EPILOG = """\ +examples: + # which GPUs, and how many, for a 70B model, 32K context, 16 users at once + throttle fit --model llama-3.3-70b --quant fp8 --context 32k --users 16 + + # with your prices, and what you run today + throttle fit --model qwen2.5-72b --quant fp8 --context 16k --users 32 \\ + --gpu h100 --gpu a100-80 --price h100=2.49 --price a100-80=1.79 --current h100=4 + + # any model: point at its config.json and give the parameter count + throttle fit --config ./config.json --params 32.8 --quant awq --context 8k --users 8 + + # llama.cpp / GGUF on your own cards (any card count works there) + throttle fit --model qwen2.5-32b --quant q4_k_m --engine llamacpp --gpu rtx3090 --gpu 1060=6 + +Built-in models: see --list-models. Built-in GPUs: see --list-gpus. +""" + + +@dataclass(frozen=True) +class ModelSpec: + name: str + params_b: float # total parameters, billions (all experts for MoE) + layers: int + kv_heads: int + head_dim: int + attention_heads: int + hf_id: str = "" # name to serve it by + + +# Published architecture numbers (each model's config.json). +MODELS: dict[str, ModelSpec] = { + spec.name: spec + for spec in ( + ModelSpec("llama-3.1-8b", 8.03, 32, 8, 128, 32, "meta-llama/Llama-3.1-8B-Instruct"), + ModelSpec("llama-3.1-70b", 70.6, 80, 8, 128, 64, "meta-llama/Llama-3.1-70B-Instruct"), + ModelSpec("llama-3.3-70b", 70.6, 80, 8, 128, 64, "meta-llama/Llama-3.3-70B-Instruct"), + ModelSpec("qwen2.5-7b", 7.62, 28, 4, 128, 28, "Qwen/Qwen2.5-7B-Instruct"), + ModelSpec("qwen2.5-14b", 14.8, 48, 8, 128, 40, "Qwen/Qwen2.5-14B-Instruct"), + ModelSpec("qwen2.5-32b", 32.8, 64, 8, 128, 40, "Qwen/Qwen2.5-32B-Instruct"), + ModelSpec("qwen2.5-72b", 72.7, 80, 8, 128, 64, "Qwen/Qwen2.5-72B-Instruct"), + ModelSpec("qwen3-32b", 32.8, 64, 8, 128, 64, "Qwen/Qwen3-32B"), + ModelSpec("qwen3-30b-a3b", 30.5, 48, 4, 128, 32, "Qwen/Qwen3-30B-A3B"), + ModelSpec("mistral-7b", 7.25, 32, 8, 128, 32, "mistralai/Mistral-7B-Instruct-v0.3"), + ModelSpec("mixtral-8x7b", 46.7, 32, 8, 128, 32, "mistralai/Mixtral-8x7B-Instruct-v0.1"), + ) +} + +# Memory on the card, GB as sold. +GPUS: dict[str, float] = { + "h200": 141, "h100": 80, "a100-80": 80, "a100-40": 40, "mi300x": 192, + "l40s": 48, "rtx6000-ada": 48, "a10g": 24, "l4": 24, + "rtx5090": 32, "rtx4090": 24, "rtx3090": 24, +} +DEFAULT_GPUS = ("h100", "a100-80", "l40s", "l4") + +# Approximate bytes per weight, including quantization scales. +QUANT_BYTES: dict[str, float] = { + "bf16": 2.0, "fp16": 2.0, "fp8": 1.0, "int8": 1.0, + "awq": 0.56, "gptq": 0.56, "int4": 0.56, + "q8_0": 1.0625, "q6_k": 0.82, "q5_k_m": 0.71, "q4_k_m": 0.60, +} +KV_BYTES: dict[str, float] = {"auto": 2.0, "fp16": 2.0, "fp8": 1.0, "q8_0": 1.0625, "q4_0": 0.5625} + + +@dataclass +class Option: + gpu: str + vram_gb: float + gpus: int | None # fewest that fit; None if not within MAX_GPUS + max_users: int | None # full-context chats that many cards can hold + monthly_cost: float | None # only when the user gave a price + hourly_price: float | None + + +def parse_size(text: str) -> int: + """'32k' -> 32768, '128K' -> 131072, '8192' -> 8192.""" + raw = text.strip().lower() + mult = 1 + if raw.endswith("k"): + raw, mult = raw[:-1], 1024 + elif raw.endswith("m"): + raw, mult = raw[:-1], 1024 * 1024 + try: + value = int(float(raw) * mult) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"not a size: {text!r} (try 8192, 32k)") from exc + if value <= 0: + raise argparse.ArgumentTypeError("must be above 0") + return value + + +def positive_int(text: str) -> int: + try: + value = int(text) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"not a whole number: {text!r}") from exc + if value <= 0: + raise argparse.ArgumentTypeError("must be above 0") + return value + + +def add_fit_arguments(parser: argparse.ArgumentParser) -> None: + which = parser.add_argument_group("the model") + which.add_argument("--model", help="a built-in model name (see --list-models)") + which.add_argument("--config", type=Path, + help="a Hugging Face config.json to read layers and heads from") + which.add_argument("--params", type=float, + help="total parameters in billions (needed with --config, " + "or to override a built-in)") + which.add_argument("--quant", default="bf16", choices=sorted(QUANT_BYTES), + help="weight format (default bf16)") + which.add_argument("--kv-dtype", default="auto", choices=sorted(KV_BYTES), + help="KV cache format (default auto = 16 bit)") + + load = parser.add_argument_group("your traffic") + load.add_argument("--context", type=parse_size, default=8192, + help="tokens each chat may grow to, prompt plus answer (default 8k)") + load.add_argument("--users", type=positive_int, default=1, + help="chats running at the same time at that length (default 1)") + + hw = parser.add_argument_group("hardware and prices") + hw.add_argument("--gpu", action="append", metavar="NAME[=GB]", + help="GPU to consider; repeat for more. A custom card: name=GB " + f"(default: {', '.join(DEFAULT_GPUS)})") + hw.add_argument("--price", action="append", metavar="NAME=$/HOUR", + help="your hourly price for one card, e.g. h100=2.49; repeat per GPU") + hw.add_argument("--current", metavar="NAME=COUNT", + help="what you run today, e.g. h100=4, to see if fewer cards look enough") + hw.add_argument("--engine", default="vllm", choices=("vllm", "sglang", "llamacpp"), + help="vllm/sglang split across 1, 2, 4 or 8 cards that divide the " + "attention heads; llamacpp can use any count (default vllm)") + hw.add_argument("--memory-fraction", type=float, default=DEFAULT_MEMORY_FRACTION, + help=f"share of each card the engine may use (default {DEFAULT_MEMORY_FRACTION})") + hw.add_argument("--overhead-gb", type=float, default=DEFAULT_OVERHEAD_GB, + help=f"GB per card kept for the runtime itself (default {DEFAULT_OVERHEAD_GB})") + + out = parser.add_argument_group("output") + out.add_argument("--json", action="store_true", help="print JSON instead of a table") + out.add_argument("--list-models", action="store_true", help="list built-in models and exit") + out.add_argument("--list-gpus", action="store_true", help="list built-in GPUs and exit") + + +class UsageError(Exception): + pass + + +def resolve_model(args: argparse.Namespace) -> ModelSpec: + if args.model and args.config: + raise UsageError("give --model or --config, not both") + if args.config: + try: + cfg = json.loads(args.config.read_text()) + except (OSError, ValueError) as exc: + raise UsageError(f"could not read {args.config}: {exc}") from exc + cfg = cfg.get("text_config", cfg) + try: + layers = int(cfg["num_hidden_layers"]) + heads = int(cfg["num_attention_heads"]) + kv_heads = int(cfg.get("num_key_value_heads") or heads) + head_dim = int(cfg.get("head_dim") or int(cfg["hidden_size"]) // heads) + except (KeyError, TypeError, ValueError) as exc: + raise UsageError(f"{args.config} is missing {exc}; is it a transformer config.json?") from exc + if not args.params: + raise UsageError("--config needs --params (total parameters in billions)") + return ModelSpec(args.config.name, args.params, layers, kv_heads, head_dim, heads) + if not args.model: + raise UsageError("give --model (see --list-models) or --config path/to/config.json") + key = args.model.strip().lower() + if key not in MODELS: + raise UsageError(f"unknown model {args.model!r}. Built in: {', '.join(MODELS)}. " + "For anything else use --config and --params.") + spec = MODELS[key] + if args.params: + spec = ModelSpec(spec.name, args.params, spec.layers, spec.kv_heads, spec.head_dim, + spec.attention_heads, spec.hf_id) + return spec + + +def parse_pairs(values: Sequence[str] | None, flag: str) -> dict[str, float]: + pairs: dict[str, float] = {} + for item in values or (): + name, sep, number = item.partition("=") + try: + value = float(number) + except ValueError: + value = -1.0 + if not sep or not name.strip() or value <= 0: + raise UsageError(f"{flag} wants NAME=NUMBER above 0, got {item!r}") + pairs[name.strip().lower()] = value + return pairs + + +def resolve_gpus(values: Sequence[str] | None) -> dict[str, float]: + gpus: dict[str, float] = {} + for item in values or DEFAULT_GPUS: + name, sep, gb = item.partition("=") + key = name.strip().lower() + if sep: + gpus.update(parse_pairs([f"{key}={gb}"], "--gpu")) + elif key in GPUS: + gpus[key] = GPUS[key] + else: + raise UsageError(f"unknown GPU {name!r}. Built in: {', '.join(GPUS)}. " + "For another card use name=GB, e.g. --gpu 1060=6") + return gpus + + +def gpu_counts(engine: str, spec: ModelSpec) -> list[int]: + if engine == "llamacpp": + return list(range(1, MAX_GPUS + 1)) + return [n for n in (1, 2, 4, 8) + if spec.attention_heads % n == 0 and (spec.kv_heads % n == 0 or n % spec.kv_heads == 0)] + + +def weights_gb(spec: ModelSpec, quant: str) -> float: + return spec.params_b * 1e9 * QUANT_BYTES[quant] / GIB + + +def kv_gb_per_token(spec: ModelSpec, kv_dtype: str) -> float: + # keys and values, every layer, every KV head + return 2 * spec.layers * spec.kv_heads * spec.head_dim * KV_BYTES[kv_dtype] / GIB + + +def usable_gb(vram_gb: float, fraction: float, overhead: float) -> float: + # cards are sold in GB (10^9); work in GiB like the engines report + return vram_gb * 1e9 / GIB * fraction - overhead + + +def max_users(n: int, per_card: float, weights: float, kv_chat: float, + kv_heads: int = 0) -> int: + room = n * per_card - weights + if room <= 0 or n * per_card <= 0: + return 0 + # split across more cards than KV heads, each head is copied to several cards + copies = max(1.0, n / kv_heads) if kv_heads else 1.0 + return int(math.floor(room / (kv_chat * copies))) + + +def plan(spec: ModelSpec, args: argparse.Namespace, gpus: Mapping[str, float], + prices: Mapping[str, float]) -> list[Option]: + weights = weights_gb(spec, args.quant) + kv_chat = kv_gb_per_token(spec, args.kv_dtype) * args.context + counts = gpu_counts(args.engine, spec) + options = [] + for name, vram in gpus.items(): + per_card = usable_gb(vram, args.memory_fraction, args.overhead_gb) + heads = 0 if args.engine == "llamacpp" else spec.kv_heads + fewest = next((n for n in counts + if max_users(n, per_card, weights, kv_chat, heads) >= args.users), None) + price = prices.get(name) + options.append(Option( + gpu=name, vram_gb=vram, gpus=fewest, + max_users=max_users(fewest, per_card, weights, kv_chat, heads) if fewest else None, + monthly_cost=round(fewest * price * HOURS_PER_MONTH, 2) if fewest and price else None, + hourly_price=price, + )) + return options + + +def pick(options: Sequence[Option]) -> Option | None: + fits = [o for o in options if o.gpus] + if not fits: + return None + priced = [o for o in fits if o.monthly_cost is not None] + if priced: + return min(priced, key=lambda o: o.monthly_cost) # type: ignore[arg-type,return-value] + return min(fits, key=lambda o: (o.gpus, o.vram_gb)) + + +def current_finding(current: tuple[str, int] | None, options: Sequence[Option]) -> dict[str, Any] | None: + if not current: + return None + name, count = current + match = next((o for o in options if o.gpu == name), None) + if match is None: + return {"gpu": name, "count": count, "status": "not_considered"} + if not match.gpus: + return {"gpu": name, "count": count, "status": "does_not_fit_estimate"} + spare = count - match.gpus + finding: dict[str, Any] = {"gpu": name, "count": count, "fewest_estimate": match.gpus, + "spare_cards": max(spare, 0), + "status": "maybe_fewer" if spare > 0 else "no_fewer"} + if spare > 0 and match.hourly_price: + finding["monthly_saving_if_confirmed"] = round(spare * match.hourly_price * HOURS_PER_MONTH, 2) + return finding + + +def measure_command(spec: ModelSpec, best: Option, current: Mapping[str, Any] | None, + engine: str) -> str: + counts = {best.gpus or 1} + if current and current.get("gpu") == best.gpu and current.get("count"): + counts.add(int(current["count"])) + model = shlex.quote(spec.hf_id or "YOUR/MODEL") + rate = f" --gpu-hourly-rate {best.hourly_price}" if best.hourly_price else "" + if engine == "llamacpp": + return (f"throttle check --url http://localhost:8000 --model {model} " + f"--gpus {best.gpus}{rate} # once per card layout you try") + launch = ("vllm serve {model} --tensor-parallel-size {tp}" if engine == "vllm" + else "python -m sglang.launch_server --model-path {model} --tp {tp}") + return (f"throttle autotune --url http://localhost:8000 --model {model}{rate} \\\n" + f" --launch \"{launch}\" \\\n" + f" --space tp={','.join(str(c) for c in sorted(counts))}") + + +def fmt_money(value: float | None) -> str: + return "-" if value is None else f"${value:,.0f}" + + +def handle_fit(args: argparse.Namespace) -> int: + if args.list_models: + for spec in MODELS.values(): + print(f"{spec.name:16} {spec.params_b:>6.1f}B {spec.layers} layers, " + f"{spec.kv_heads} KV heads x {spec.head_dim}") + return EXIT_OK + if args.list_gpus: + for name, gb in GPUS.items(): + print(f"{name:12} {gb:>4.0f} GB") + return EXIT_OK + try: + if not 0 < args.memory_fraction <= 1: + raise UsageError("--memory-fraction must be above 0 and at most 1") + if args.overhead_gb < 0: + raise UsageError("--overhead-gb cannot be negative") + spec = resolve_model(args) + gpus = resolve_gpus(args.gpu) + prices = parse_pairs(args.price, "--price") + unknown = sorted(set(prices) - set(gpus)) + if unknown: + raise UsageError(f"--price for {', '.join(unknown)} but that GPU is not in the list; " + "add it with --gpu") + current = None + if args.current: + cur = parse_pairs([args.current], "--current") + (cur_name, cur_count), = cur.items() + if cur_count != int(cur_count): + raise UsageError("--current wants a whole card count, e.g. h100=4") + current = (cur_name, int(cur_count)) + if cur_name not in gpus: + gpus = {**gpus, **resolve_gpus([cur_name])} + except UsageError as exc: + print(f"Error: {exc}", file=sys.stderr) + return EXIT_USAGE + + options = plan(spec, args, gpus, prices) + best = pick(options) + finding = current_finding(current, options) + weights = weights_gb(spec, args.quant) + kv_chat = kv_gb_per_token(spec, args.kv_dtype) * args.context + command = measure_command(spec, best, finding, args.engine) if best else None + + if args.json: + print(json.dumps({ + "estimate": True, + "model": asdict(spec), "quant": args.quant, "kv_dtype": args.kv_dtype, + "context": args.context, "users": args.users, "engine": args.engine, + "weights_gib": round(weights, 2), "kv_gib_per_chat": round(kv_chat, 3), + "options": [asdict(o) for o in options], + "pick": best.gpu if best else None, "current": finding, + "measure_command": command, + }, indent=2)) + return EXIT_OK if best else EXIT_DOES_NOT_FIT + + st = style_module.Style() + print(st.rule("throttle fit (ESTIMATE: memory math, not measured)")) + print(st.dim("Assumes every chat at the full --context at once (the safe side).")) + print(f"{spec.name}, {args.quant} weights, {args.context:,} token context, " + f"{args.users} chat{'s' if args.users != 1 else ''} at once, {args.engine}") + print(st.dim(f"weights about {weights:,.1f} GiB, KV cache about {kv_chat:,.2f} GiB per full chat")) + print() + header = f" {'GPU':12} {'memory':>7} {'fewest cards':>13} {'chats they hold':>16} {'per month':>10}" + print(st.bold(header)) + for o in options: + cards = str(o.gpus) if o.gpus else f">{MAX_GPUS}" if args.engine == "llamacpp" else "no fit" + users = str(o.max_users) if o.max_users is not None else "-" + line = (f" {o.gpu:12} {o.vram_gb:>5.0f}GB {cards:>13} {users:>16} " + f"{fmt_money(o.monthly_cost):>10}") + print(st.good(line) if best and o is best else line) + print() + + if not best: + print(st.bad(f"Nothing here fits on up to {MAX_GPUS} cards.")) + print("Try a smaller weight format (--quant fp8 or awq), a fp8 KV cache " + "(--kv-dtype fp8), a shorter --context, fewer --users, or a bigger GPU.") + return EXIT_DOES_NOT_FIT + + why = "cheapest at your prices" if best.monthly_cost is not None else "fewest cards" + print(f"Best looking: {st.bold(f'{best.gpus} x {best.gpu}')} ({why}).") + if best.monthly_cost is None: + print(st.dim("Add --price NAME=$/HOUR for the monthly cost of each option.")) + if finding: + status = finding["status"] + if status == "maybe_fewer": + line = (f"You run {finding['count']} x {finding['gpu']}; the estimate says " + f"{finding['fewest_estimate']} may be enough.") + if "monthly_saving_if_confirmed" in finding: + line += (f" If a measurement agrees, that is about " + f"{fmt_money(finding['monthly_saving_if_confirmed'])} a month.") + print(st.warn(line)) + elif status == "no_fewer": + print(f"You run {finding['count']} x {finding['gpu']}; the estimate does not " + "find room to drop a card at this load.") + elif status == "does_not_fit_estimate": + print(st.warn(f"The estimate says {finding['gpu']} cannot hold this load on up to " + f"{MAX_GPUS} cards, yet you run it: your real chats are probably " + "shorter than --context, or the numbers above need your config.json.")) + print() + print("This does not tell you speed. Extra cards add traffic between them and") + print("can be slower. Measure on your server before you change anything:") + print(st.accent(f" {command}")) + return EXIT_OK diff --git a/tests/test_cli.py b/tests/test_cli.py index aac7221..86387dd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -410,6 +410,7 @@ def test_parser_subcommands_match_expected(self) -> None: "autotune", "gate", "coldstart", + "fit", "capture", "shadow", "eval", diff --git a/tests/test_fit.py b/tests/test_fit.py new file mode 100644 index 0000000..05effe4 --- /dev/null +++ b/tests/test_fit.py @@ -0,0 +1,141 @@ +"""`throttle fit`: offline memory estimate of the fewest GPUs for a model.""" + +from __future__ import annotations + +import json + +import pytest + +import throttle.fit as fit +from throttle.cli import main + + +def run(capsys, *argv: str) -> tuple[int, str, str]: + code = main(["fit", *argv]) + out, err = capsys.readouterr() + return code, out, err + + +def test_kv_math_matches_hand_calculation(): + spec = fit.MODELS["llama-3.3-70b"] + # 2 (K and V) x 80 layers x 8 heads x 128 dims x 2 bytes = 327,680 bytes/token + assert fit.kv_gb_per_token(spec, "auto") * fit.GIB == pytest.approx(327_680) + assert fit.kv_gb_per_token(spec, "auto") * 32768 == pytest.approx(10.0) + + +def test_weights_scale_with_quant(): + spec = fit.MODELS["qwen2.5-72b"] + assert fit.weights_gb(spec, "fp8") == pytest.approx(fit.weights_gb(spec, "bf16") / 2) + + +def test_fewest_h100s_for_70b_fp8(capsys): + code, out, _ = run(capsys, "--model", "llama-3.3-70b", "--quant", "fp8", + "--context", "32k", "--users", "16", "--gpu", "h100", "--json") + assert code == 0 + data = json.loads(out) + assert data["estimate"] is True + (h100,) = data["options"] + # 65.8 GiB weights + 16 x 10 GiB KV needs more than 2 cards, fits on 4 + assert h100["gpus"] == 4 + assert h100["max_users"] >= 16 + + +def test_current_flags_spare_cards_with_saving(capsys): + code, out, _ = run(capsys, "--model", "llama-3.3-70b", "--quant", "fp8", + "--context", "8k", "--users", "16", "--gpu", "h100", + "--price", "h100=2.00", "--current", "h100=4", "--json") + assert code == 0 + data = json.loads(out) + assert data["current"]["status"] == "maybe_fewer" + assert data["current"]["fewest_estimate"] == 2 + assert data["current"]["monthly_saving_if_confirmed"] == pytest.approx(2 * 2.00 * 730) + assert "--space tp=2,4" in data["measure_command"] + assert "meta-llama/Llama-3.3-70B-Instruct" in data["measure_command"] + + +def test_current_on_other_gpu_is_not_mixed_into_space(capsys): + _, out, _ = run(capsys, "--model", "llama-3.3-70b", "--quant", "fp8", "--context", "32k", + "--users", "16", "--gpu", "l40s", "--price", "l40s=1", + "--current", "h100=4", "--json") + data = json.loads(out) + assert data["pick"] == "l40s" + assert "--space tp=8" in data["measure_command"] + + +def test_vllm_counts_respect_attention_heads(): + # Qwen2.5-7B has 28 attention heads: 8-way tensor parallel is not possible + assert fit.gpu_counts("vllm", fit.MODELS["qwen2.5-7b"]) == [1, 2, 4] + assert fit.gpu_counts("llamacpp", fit.MODELS["qwen2.5-7b"]) == list(range(1, 9)) + + +def test_kv_copied_when_more_cards_than_kv_heads(): + one = fit.max_users(8, 70.0, 10.0, 1.0, kv_heads=4) + none = fit.max_users(8, 70.0, 10.0, 1.0, kv_heads=0) + assert one == none // 2 + + +def test_does_not_fit_exits_1_with_advice(capsys): + code, out, _ = run(capsys, "--model", "qwen2.5-72b", "--context", "128k", + "--users", "64", "--gpu", "l4") + assert code == 1 + assert "Nothing here fits" in out + assert "--quant" in out + + +def test_table_says_estimate_and_points_to_measurement(capsys): + code, out, _ = run(capsys, "--model", "qwen2.5-7b", "--users", "4") + assert code == 0 + assert "ESTIMATE" in out + assert "throttle autotune" in out + assert "Qwen/Qwen2.5-7B-Instruct" in out + + +def test_custom_gpu_and_llamacpp(capsys): + code, out, _ = run(capsys, "--model", "qwen2.5-32b", "--quant", "q4_k_m", + "--engine", "llamacpp", "--gpu", "rtx3090", "--gpu", "1060=6", + "--context", "32k", "--users", "2", "--json") + assert code == 0 + data = json.loads(out) + by_name = {o["gpu"]: o for o in data["options"]} + assert by_name["1060"]["vram_gb"] == 6 + assert by_name["rtx3090"]["gpus"] == 2 + assert data["measure_command"].startswith("throttle check") + + +def test_config_json(tmp_path, capsys): + cfg = tmp_path / "config.json" + cfg.write_text(json.dumps({"num_hidden_layers": 32, "num_attention_heads": 32, + "num_key_value_heads": 8, "hidden_size": 4096})) + code, out, _ = run(capsys, "--config", str(cfg), "--params", "8", "--json") + assert code == 0 + data = json.loads(out) + assert data["model"]["head_dim"] == 128 + assert "YOUR/MODEL" in data["measure_command"] + + +@pytest.mark.parametrize("argv, message", [ + ((), "give --model"), + (("--model", "nope"), "unknown model"), + (("--model", "qwen2.5-7b", "--gpu", "b9000"), "unknown GPU"), + (("--model", "qwen2.5-7b", "--price", "h100=abc"), "--price wants"), + (("--model", "qwen2.5-7b", "--gpu", "h100", "--price", "l4=1"), "not in the list"), + (("--model", "qwen2.5-7b", "--current", "h100=1.5"), "whole card count"), + (("--config", "missing.json", "--params", "7"), "could not read"), +]) +def test_usage_errors(capsys, argv, message): + code, _, err = run(capsys, *argv) + assert code == 2 + assert message in err + + +def test_bad_context_is_rejected_by_argparse(capsys): + with pytest.raises(SystemExit) as exc: + main(["fit", "--model", "qwen2.5-7b", "--context", "lots"]) + assert exc.value.code == 2 + + +def test_lists(capsys): + assert main(["fit", "--list-models"]) == 0 + assert "llama-3.3-70b" in capsys.readouterr().out + assert main(["fit", "--list-gpus"]) == 0 + assert "h100" in capsys.readouterr().out From 6f25b780bfb2f837791483b7a7a743f29928a26b Mon Sep 17 00:00:00 2001 From: Kushagra Kanaujia Date: Mon, 16 Nov 2026 22:48:46 -0800 Subject: [PATCH 2/2] fit: fix review findings Always give autotune a rate (placeholder when unpriced); total rate for llama.cpp check; only legal tp counts in --space and never a single value; measure the user's own card when --current is given, and never pick a GPU that only came in through --current; needs_more status; reject inf/nan and negative params; keep embeddings 16 bit for fp8/AWQ/GPTQ; treat sold GB as GiB; pass context, fp8 and concurrency into the measure command; Qwen2.5 14B/32B parameter counts. Co-Authored-By: Claude Opus 5.5 --- .../console/docs/guides/fewest-gpus.md | 8 +- src/throttle/fit.py | 148 ++++++++++++------ tests/test_fit.py | 60 ++++++- 3 files changed, 161 insertions(+), 55 deletions(-) diff --git a/src/throttle/console/docs/guides/fewest-gpus.md b/src/throttle/console/docs/guides/fewest-gpus.md index eeb2fb2..8b8f5e7 100644 --- a/src/throttle/console/docs/guides/fewest-gpus.md +++ b/src/throttle/console/docs/guides/fewest-gpus.md @@ -23,6 +23,7 @@ With `--current`, it tells you if fewer cards look like enough, and what that wo - **Speed.** More cards can be slower, because they have to talk to each other. Measure before you change anything. - **Your real chat lengths.** It assumes every chat at the full `--context` at the same time. Real traffic is usually lighter, so this errs on the safe side. +- **Weight sizes are approximate.** fp8, AWQ and GPTQ keep the embeddings in 16 bit, which is counted; other layers left unquantized are not. - **Exotic attention.** Sliding-window and latent attention models need less KV memory than this math assumes. ### Then measure it @@ -31,11 +32,12 @@ With `--current`, it tells you if fewer cards look like enough, and what that wo ```bash throttle autotune --url http://localhost:8000 --model meta-llama/Llama-3.3-70B-Instruct \ - --gpu-hourly-rate 2.49 \ - --launch "vllm serve {model} --tensor-parallel-size {tp}" --space tp=2,4 + --gpu-hourly-rate 2.49 --concurrency 16 \ + --launch "vllm serve {model} --tensor-parallel-size {tp} --max-model-len 32768 --quantization fp8" \ + --space tp=2,4 ``` -That gives you real cost per million tokens for each card count, with confidence intervals. +It measures cost per million tokens on your server for each card count, with a range that shows how sure the measurement is. Run it on the machine type you actually rent; replace `PRICE_PER_GPU_HOUR` if you did not give `--price`. ### Any model, any card diff --git a/src/throttle/fit.py b/src/throttle/fit.py index afa836d..66c5861 100644 --- a/src/throttle/fit.py +++ b/src/throttle/fit.py @@ -37,7 +37,7 @@ # What vLLM lets itself use by default (--gpu-memory-utilization 0.9), and a # flat allowance per card for the CUDA context, activations and graphs. DEFAULT_MEMORY_FRACTION = 0.90 -DEFAULT_OVERHEAD_GB = 1.5 +DEFAULT_OVERHEAD_GB = 2.0 MAX_GPUS = 8 FIT_DESCRIPTION = __doc__.split("\n\n", 1)[1] if __doc__ else "" @@ -69,27 +69,30 @@ class ModelSpec: head_dim: int attention_heads: int hf_id: str = "" # name to serve it by + vocab: int = 0 # with hidden: embeddings and lm_head, which + hidden: int = 0 # fp8/AWQ/GPTQ checkpoints keep in 16 bit # Published architecture numbers (each model's config.json). MODELS: dict[str, ModelSpec] = { spec.name: spec for spec in ( - ModelSpec("llama-3.1-8b", 8.03, 32, 8, 128, 32, "meta-llama/Llama-3.1-8B-Instruct"), - ModelSpec("llama-3.1-70b", 70.6, 80, 8, 128, 64, "meta-llama/Llama-3.1-70B-Instruct"), - ModelSpec("llama-3.3-70b", 70.6, 80, 8, 128, 64, "meta-llama/Llama-3.3-70B-Instruct"), - ModelSpec("qwen2.5-7b", 7.62, 28, 4, 128, 28, "Qwen/Qwen2.5-7B-Instruct"), - ModelSpec("qwen2.5-14b", 14.8, 48, 8, 128, 40, "Qwen/Qwen2.5-14B-Instruct"), - ModelSpec("qwen2.5-32b", 32.8, 64, 8, 128, 40, "Qwen/Qwen2.5-32B-Instruct"), - ModelSpec("qwen2.5-72b", 72.7, 80, 8, 128, 64, "Qwen/Qwen2.5-72B-Instruct"), - ModelSpec("qwen3-32b", 32.8, 64, 8, 128, 64, "Qwen/Qwen3-32B"), - ModelSpec("qwen3-30b-a3b", 30.5, 48, 4, 128, 32, "Qwen/Qwen3-30B-A3B"), - ModelSpec("mistral-7b", 7.25, 32, 8, 128, 32, "mistralai/Mistral-7B-Instruct-v0.3"), - ModelSpec("mixtral-8x7b", 46.7, 32, 8, 128, 32, "mistralai/Mixtral-8x7B-Instruct-v0.1"), + ModelSpec("llama-3.1-8b", 8.03, 32, 8, 128, 32, "meta-llama/Llama-3.1-8B-Instruct", 128256, 4096), + ModelSpec("llama-3.1-70b", 70.6, 80, 8, 128, 64, "meta-llama/Llama-3.1-70B-Instruct", 128256, 8192), + ModelSpec("llama-3.3-70b", 70.6, 80, 8, 128, 64, "meta-llama/Llama-3.3-70B-Instruct", 128256, 8192), + ModelSpec("qwen2.5-7b", 7.62, 28, 4, 128, 28, "Qwen/Qwen2.5-7B-Instruct", 152064, 3584), + ModelSpec("qwen2.5-14b", 14.7, 48, 8, 128, 40, "Qwen/Qwen2.5-14B-Instruct", 152064, 5120), + ModelSpec("qwen2.5-32b", 32.5, 64, 8, 128, 40, "Qwen/Qwen2.5-32B-Instruct", 152064, 5120), + ModelSpec("qwen2.5-72b", 72.7, 80, 8, 128, 64, "Qwen/Qwen2.5-72B-Instruct", 152064, 8192), + ModelSpec("qwen3-32b", 32.8, 64, 8, 128, 64, "Qwen/Qwen3-32B", 151936, 5120), + ModelSpec("qwen3-30b-a3b", 30.5, 48, 4, 128, 32, "Qwen/Qwen3-30B-A3B", 151936, 2048), + ModelSpec("mistral-7b", 7.25, 32, 8, 128, 32, "mistralai/Mistral-7B-Instruct-v0.3", 32768, 4096), + ModelSpec("mixtral-8x7b", 46.7, 32, 8, 128, 32, "mistralai/Mixtral-8x7B-Instruct-v0.1", 32000, 4096), ) } -# Memory on the card, GB as sold. +# Memory on the card as sold. NVIDIA's "80GB" is about 79.6 GiB usable, so +# the sold figure is treated as GiB. GPUS: dict[str, float] = { "h200": 141, "h100": 80, "a100-80": 80, "a100-40": 40, "mi300x": 192, "l40s": 48, "rtx6000-ada": 48, "a10g": 24, "l4": 24, @@ -125,7 +128,10 @@ def parse_size(text: str) -> int: elif raw.endswith("m"): raw, mult = raw[:-1], 1024 * 1024 try: - value = int(float(raw) * mult) + number = float(raw) + if not math.isfinite(number): + raise ValueError(raw) + value = int(number * mult) except ValueError as exc: raise argparse.ArgumentTypeError(f"not a size: {text!r} (try 8192, 32k)") from exc if value <= 0: @@ -189,6 +195,8 @@ class UsageError(Exception): def resolve_model(args: argparse.Namespace) -> ModelSpec: + if args.params is not None and (not math.isfinite(args.params) or args.params <= 0): + raise UsageError("--params must be a number of billions above 0") if args.model and args.config: raise UsageError("give --model or --config, not both") if args.config: @@ -206,7 +214,12 @@ def resolve_model(args: argparse.Namespace) -> ModelSpec: raise UsageError(f"{args.config} is missing {exc}; is it a transformer config.json?") from exc if not args.params: raise UsageError("--config needs --params (total parameters in billions)") - return ModelSpec(args.config.name, args.params, layers, kv_heads, head_dim, heads) + vocab = int(cfg.get("vocab_size") or 0) + hidden = int(cfg.get("hidden_size") or 0) + if cfg.get("tie_word_embeddings"): + hidden //= 2 # one shared matrix, not two: weights_gb counts vocab x hidden x 2 + return ModelSpec(args.config.name, args.params, layers, kv_heads, head_dim, heads, + "", vocab, hidden) if not args.model: raise UsageError("give --model (see --list-models) or --config path/to/config.json") key = args.model.strip().lower() @@ -216,7 +229,7 @@ def resolve_model(args: argparse.Namespace) -> ModelSpec: spec = MODELS[key] if args.params: spec = ModelSpec(spec.name, args.params, spec.layers, spec.kv_heads, spec.head_dim, - spec.attention_heads, spec.hf_id) + spec.attention_heads, spec.hf_id, spec.vocab, spec.hidden) return spec @@ -228,7 +241,7 @@ def parse_pairs(values: Sequence[str] | None, flag: str) -> dict[str, float]: value = float(number) except ValueError: value = -1.0 - if not sep or not name.strip() or value <= 0: + if not sep or not name.strip() or not math.isfinite(value) or value <= 0: raise UsageError(f"{flag} wants NAME=NUMBER above 0, got {item!r}") pairs[name.strip().lower()] = value return pairs @@ -256,8 +269,16 @@ def gpu_counts(engine: str, spec: ModelSpec) -> list[int]: if spec.attention_heads % n == 0 and (spec.kv_heads % n == 0 or n % spec.kv_heads == 0)] +KEEPS_EMBEDDINGS_16BIT = {"fp8", "int8", "awq", "gptq", "int4"} + + def weights_gb(spec: ModelSpec, quant: str) -> float: - return spec.params_b * 1e9 * QUANT_BYTES[quant] / GIB + total = spec.params_b * 1e9 + kept = 0.0 + if quant in KEEPS_EMBEDDINGS_16BIT: + # input embeddings plus lm_head + kept = min(total, 2.0 * spec.vocab * spec.hidden) + return ((total - kept) * QUANT_BYTES[quant] + kept * 2.0) / GIB def kv_gb_per_token(spec: ModelSpec, kv_dtype: str) -> float: @@ -266,8 +287,7 @@ def kv_gb_per_token(spec: ModelSpec, kv_dtype: str) -> float: def usable_gb(vram_gb: float, fraction: float, overhead: float) -> float: - # cards are sold in GB (10^9); work in GiB like the engines report - return vram_gb * 1e9 / GIB * fraction - overhead + return vram_gb * fraction - overhead def max_users(n: int, per_card: float, weights: float, kv_chat: float, @@ -301,14 +321,13 @@ def plan(spec: ModelSpec, args: argparse.Namespace, gpus: Mapping[str, float], return options -def pick(options: Sequence[Option]) -> Option | None: - fits = [o for o in options if o.gpus] +def pick(options: Sequence[Option], listed: set[str]) -> tuple[Option | None, str]: + fits = [o for o in options if o.gpus and o.gpu in listed] if not fits: - return None - priced = [o for o in fits if o.monthly_cost is not None] - if priced: - return min(priced, key=lambda o: o.monthly_cost) # type: ignore[arg-type,return-value] - return min(fits, key=lambda o: (o.gpus, o.vram_gb)) + return None, "" + if all(o.monthly_cost is not None for o in fits): + return min(fits, key=lambda o: o.monthly_cost), "cheapest at your prices" # type: ignore[arg-type,return-value] + return min(fits, key=lambda o: (o.gpus, o.vram_gb)), "fewest cards" def current_finding(current: tuple[str, int] | None, options: Sequence[Option]) -> dict[str, Any] | None: @@ -323,27 +342,56 @@ def current_finding(current: tuple[str, int] | None, options: Sequence[Option]) spare = count - match.gpus finding: dict[str, Any] = {"gpu": name, "count": count, "fewest_estimate": match.gpus, "spare_cards": max(spare, 0), - "status": "maybe_fewer" if spare > 0 else "no_fewer"} + "status": ("maybe_fewer" if spare > 0 else + "needs_more" if spare < 0 else "no_fewer")} if spare > 0 and match.hourly_price: finding["monthly_saving_if_confirmed"] = round(spare * match.hourly_price * HOURS_PER_MONTH, 2) return finding -def measure_command(spec: ModelSpec, best: Option, current: Mapping[str, Any] | None, - engine: str) -> str: - counts = {best.gpus or 1} - if current and current.get("gpu") == best.gpu and current.get("count"): - counts.add(int(current["count"])) +def measure_command(spec: ModelSpec, args: argparse.Namespace, options: Sequence[Option], + best: Option, current: Mapping[str, Any] | None) -> str: + legal = gpu_counts(args.engine, spec) + target = best + counts: set[int] = {best.gpus or 1} + if current and current.get("fewest_estimate"): + mine = next((o for o in options if o.gpu == current["gpu"]), None) + if mine and mine.gpus: + target = mine + counts = {mine.gpus} + if current["count"] in legal: + counts.add(int(current["count"])) + if len(counts) == 1: + above = [n for n in legal if n > next(iter(counts))] + if above: + counts.add(above[0]) model = shlex.quote(spec.hf_id or "YOUR/MODEL") - rate = f" --gpu-hourly-rate {best.hourly_price}" if best.hourly_price else "" - if engine == "llamacpp": - return (f"throttle check --url http://localhost:8000 --model {model} " - f"--gpus {best.gpus}{rate} # once per card layout you try") - launch = ("vllm serve {model} --tensor-parallel-size {tp}" if engine == "vllm" - else "python -m sglang.launch_server --model-path {model} --tp {tp}") - return (f"throttle autotune --url http://localhost:8000 --model {model}{rate} \\\n" + if args.engine == "llamacpp": + n = target.gpus or 1 + total = (f"{target.hourly_price * n:g}" if target.hourly_price + else "TOTAL_PRICE_PER_HOUR") + return (f"throttle check --url http://localhost:8000 --model {model} --gpus {n} " + f"--gpu-hourly-rate {total} --concurrency {args.users}" + f" # on {n} x {target.gpu}; rate is for all {n} cards") + rate = f"{target.hourly_price:g}" if target.hourly_price else "PRICE_PER_GPU_HOUR" + if args.engine == "vllm": + launch = f"vllm serve {{model}} --tensor-parallel-size {{tp}} --max-model-len {args.context}" + if args.quant == "fp8": + launch += " --quantization fp8" + if args.kv_dtype == "fp8": + launch += " --kv-cache-dtype fp8" + else: + launch = (f"python -m sglang.launch_server --model-path {{model}} --tp {{tp}} " + f"--context-length {args.context}") + if args.quant == "fp8": + launch += " --quantization fp8" + if args.kv_dtype == "fp8": + launch += " --kv-cache-dtype fp8_e5m2" + return (f"throttle autotune --url http://localhost:8000 --model {model} " + f"--gpu-hourly-rate {rate} --concurrency {args.users} \\\n" f" --launch \"{launch}\" \\\n" - f" --space tp={','.join(str(c) for c in sorted(counts))}") + f" --space tp={','.join(str(c) for c in sorted(counts))}" + f" # on {'your' if target is not best or current else 'a'} {target.gpu} machine") def fmt_money(value: float | None) -> str: @@ -363,10 +411,13 @@ def handle_fit(args: argparse.Namespace) -> int: try: if not 0 < args.memory_fraction <= 1: raise UsageError("--memory-fraction must be above 0 and at most 1") + if not math.isfinite(args.memory_fraction) or not math.isfinite(args.overhead_gb): + raise UsageError("--memory-fraction and --overhead-gb must be numbers") if args.overhead_gb < 0: raise UsageError("--overhead-gb cannot be negative") spec = resolve_model(args) gpus = resolve_gpus(args.gpu) + listed = set(gpus) prices = parse_pairs(args.price, "--price") unknown = sorted(set(prices) - set(gpus)) if unknown: @@ -376,7 +427,7 @@ def handle_fit(args: argparse.Namespace) -> int: if args.current: cur = parse_pairs([args.current], "--current") (cur_name, cur_count), = cur.items() - if cur_count != int(cur_count): + if cur_count != int(cur_count) or cur_count > 64: raise UsageError("--current wants a whole card count, e.g. h100=4") current = (cur_name, int(cur_count)) if cur_name not in gpus: @@ -386,11 +437,11 @@ def handle_fit(args: argparse.Namespace) -> int: return EXIT_USAGE options = plan(spec, args, gpus, prices) - best = pick(options) + best, why = pick(options, listed) finding = current_finding(current, options) weights = weights_gb(spec, args.quant) kv_chat = kv_gb_per_token(spec, args.kv_dtype) * args.context - command = measure_command(spec, best, finding, args.engine) if best else None + command = measure_command(spec, args, options, best, finding) if best else None if args.json: print(json.dumps({ @@ -427,10 +478,9 @@ def handle_fit(args: argparse.Namespace) -> int: "(--kv-dtype fp8), a shorter --context, fewer --users, or a bigger GPU.") return EXIT_DOES_NOT_FIT - why = "cheapest at your prices" if best.monthly_cost is not None else "fewest cards" print(f"Best looking: {st.bold(f'{best.gpus} x {best.gpu}')} ({why}).") - if best.monthly_cost is None: - print(st.dim("Add --price NAME=$/HOUR for the monthly cost of each option.")) + if why == "fewest cards": + print(st.dim("Give --price NAME=$/HOUR for every GPU to pick by monthly cost.")) if finding: status = finding["status"] if status == "maybe_fewer": @@ -443,6 +493,10 @@ def handle_fit(args: argparse.Namespace) -> int: elif status == "no_fewer": print(f"You run {finding['count']} x {finding['gpu']}; the estimate does not " "find room to drop a card at this load.") + elif status == "needs_more": + print(st.warn(f"You run {finding['count']} x {finding['gpu']}, but the estimate needs " + f"{finding['fewest_estimate']} for every chat at the full --context. " + "Your real chats are probably shorter; measure to see.")) elif status == "does_not_fit_estimate": print(st.warn(f"The estimate says {finding['gpu']} cannot hold this load on up to " f"{MAX_GPUS} cards, yet you run it: your real chats are probably " diff --git a/tests/test_fit.py b/tests/test_fit.py index 05effe4..3ea04f0 100644 --- a/tests/test_fit.py +++ b/tests/test_fit.py @@ -25,7 +25,9 @@ def test_kv_math_matches_hand_calculation(): def test_weights_scale_with_quant(): spec = fit.MODELS["qwen2.5-72b"] - assert fit.weights_gb(spec, "fp8") == pytest.approx(fit.weights_gb(spec, "bf16") / 2) + half = fit.weights_gb(spec, "bf16") / 2 + # fp8 is half of bf16 except the 16 bit embeddings and lm_head + assert half < fit.weights_gb(spec, "fp8") < half * 1.05 def test_fewest_h100s_for_70b_fp8(capsys): @@ -53,13 +55,55 @@ def test_current_flags_spare_cards_with_saving(capsys): assert "meta-llama/Llama-3.3-70B-Instruct" in data["measure_command"] -def test_current_on_other_gpu_is_not_mixed_into_space(capsys): - _, out, _ = run(capsys, "--model", "llama-3.3-70b", "--quant", "fp8", "--context", "32k", +def test_current_gpu_is_measured_but_never_the_pick(capsys): + _, out, _ = run(capsys, "--model", "llama-3.3-70b", "--quant", "fp8", "--context", "8k", "--users", "16", "--gpu", "l40s", "--price", "l40s=1", "--current", "h100=4", "--json") data = json.loads(out) - assert data["pick"] == "l40s" - assert "--space tp=8" in data["measure_command"] + assert data["pick"] == "l40s" # h100 only came in through --current + assert data["current"]["status"] == "maybe_fewer" + # the command checks the claim on the user's own h100s + assert "--space tp=2,4" in data["measure_command"] + assert "h100 machine" in data["measure_command"] + + +def test_illegal_current_count_is_not_put_in_space(capsys): + _, out, _ = run(capsys, "--model", "llama-3.3-70b", "--quant", "fp8", "--context", "8k", + "--users", "16", "--gpu", "h100", "--current", "h100=3", "--json") + assert "tp=3" not in json.loads(out)["measure_command"] + + +def test_autotune_command_always_has_a_rate_and_the_context(capsys): + _, out, _ = run(capsys, "--model", "qwen2.5-7b", "--context", "16k", "--users", "4", + "--kv-dtype", "fp8", "--json") + cmd = json.loads(out)["measure_command"] + assert "--gpu-hourly-rate PRICE_PER_GPU_HOUR" in cmd + assert "--max-model-len 16384" in cmd + assert "--kv-cache-dtype fp8" in cmd + assert "--concurrency 4" in cmd + assert "--space tp=1,2" in cmd # never a single value: something to compare + + +def test_llamacpp_check_gets_the_total_rate(capsys): + _, out, _ = run(capsys, "--model", "qwen2.5-32b", "--quant", "q4_k_m", "--engine", + "llamacpp", "--gpu", "rtx3090", "--price", "rtx3090=0.4", + "--context", "32k", "--users", "2", "--json") + cmd = json.loads(out)["measure_command"] + assert "--gpus 2 --gpu-hourly-rate 0.8" in cmd + + +def test_needs_more_than_current(capsys): + _, out, _ = run(capsys, "--model", "llama-3.3-70b", "--quant", "fp8", "--context", "32k", + "--users", "16", "--gpu", "h100", "--current", "h100=2") + assert "but the estimate needs 4" in out + + +def test_quantized_checkpoints_keep_16bit_embeddings(): + spec = fit.MODELS["qwen2.5-7b"] + embed = 2 * 152064 * 3584 + expected = ((7.62e9 - embed) * 0.56 + embed * 2) / fit.GIB + assert fit.weights_gb(spec, "awq") == pytest.approx(expected) + assert fit.weights_gb(spec, "q4_k_m") == pytest.approx(7.62e9 * 0.60 / fit.GIB) def test_vllm_counts_respect_attention_heads(): @@ -121,6 +165,10 @@ def test_config_json(tmp_path, capsys): (("--model", "qwen2.5-7b", "--gpu", "h100", "--price", "l4=1"), "not in the list"), (("--model", "qwen2.5-7b", "--current", "h100=1.5"), "whole card count"), (("--config", "missing.json", "--params", "7"), "could not read"), + (("--model", "qwen2.5-7b", "--params", "-5"), "--params must"), + (("--model", "qwen2.5-7b", "--gpu", "h100", "--price", "h100=nan"), "--price wants"), + (("--model", "qwen2.5-7b", "--current", "h100=nan"), "--current wants"), + (("--model", "qwen2.5-7b", "--current", "h100=1000"), "whole card count"), ]) def test_usage_errors(capsys, argv, message): code, _, err = run(capsys, *argv) @@ -131,6 +179,8 @@ def test_usage_errors(capsys, argv, message): def test_bad_context_is_rejected_by_argparse(capsys): with pytest.raises(SystemExit) as exc: main(["fit", "--model", "qwen2.5-7b", "--context", "lots"]) + with pytest.raises(SystemExit): + main(["fit", "--model", "qwen2.5-7b", "--context", "inf"]) assert exc.value.code == 2