|
| 1 | +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | + |
| 4 | +"""Routing contract for dense Qwen3 models using TensorRT native KV cache.""" |
| 5 | + |
| 6 | +from __future__ import annotations |
| 7 | + |
| 8 | +import math |
| 9 | +import operator |
| 10 | + |
| 11 | +_INT32_MAX = (1 << 31) - 1 |
| 12 | +_UINT64_MAX = (1 << 64) - 1 |
| 13 | + |
| 14 | + |
| 15 | +class NativeKvCapability: |
| 16 | + """Small, loader-safe capability result (no dataclass dependency).""" |
| 17 | + |
| 18 | + __slots__ = ("applicable", "eligible", "reason") |
| 19 | + |
| 20 | + def __init__( |
| 21 | + self, |
| 22 | + applicable: bool, |
| 23 | + eligible: bool, |
| 24 | + reason: str, |
| 25 | + ) -> None: |
| 26 | + self.applicable = applicable |
| 27 | + self.eligible = eligible |
| 28 | + self.reason = reason |
| 29 | + |
| 30 | + |
| 31 | +def _result( |
| 32 | + *, |
| 33 | + applicable: bool = True, |
| 34 | + reasons: list[str] | tuple[str, ...] = (), |
| 35 | +) -> NativeKvCapability: |
| 36 | + return NativeKvCapability( |
| 37 | + applicable, |
| 38 | + applicable and not reasons, |
| 39 | + "; ".join(reasons) or "supported", |
| 40 | + ) |
| 41 | + |
| 42 | + |
| 43 | +def _raw(config: object) -> dict: |
| 44 | + value = getattr(config, "raw", {}) |
| 45 | + return value if isinstance(value, dict) else {} |
| 46 | + |
| 47 | + |
| 48 | +def _integer(value: object, name: str) -> int: |
| 49 | + if isinstance(value, bool): |
| 50 | + raise ValueError(f"{name} must be an integer") |
| 51 | + try: |
| 52 | + return int(operator.index(value)) |
| 53 | + except (TypeError, ValueError, OverflowError) as exc: |
| 54 | + raise ValueError(f"{name} must be an integer") from exc |
| 55 | + |
| 56 | + |
| 57 | +def _positive(config: object, name: str) -> int: |
| 58 | + value = _integer(getattr(config, name, None), name) |
| 59 | + if value <= 0: |
| 60 | + raise ValueError(f"{name} must be positive") |
| 61 | + if value > _INT32_MAX: |
| 62 | + raise ValueError(f"{name} exceeds TensorRT's int32 dimension limit") |
| 63 | + return value |
| 64 | + |
| 65 | + |
| 66 | +def resolved_head_dim(config: object) -> int: |
| 67 | + """Return the explicit HF head width, or derive it when absent.""" |
| 68 | + |
| 69 | + raw = _raw(config) |
| 70 | + explicit = raw.get("head_dim", getattr(config, "_head_dim", 0)) |
| 71 | + if "head_dim" in raw or explicit not in (None, 0): |
| 72 | + head_dim = _integer(explicit, "head_dim") |
| 73 | + else: |
| 74 | + hidden = _positive(config, "hidden_size") |
| 75 | + heads = _positive(config, "num_attention_heads") |
| 76 | + if hidden % heads: |
| 77 | + raise ValueError( |
| 78 | + "hidden_size must be divisible by num_attention_heads when head_dim is absent" |
| 79 | + ) |
| 80 | + head_dim = hidden // heads |
| 81 | + if not 0 < head_dim <= _INT32_MAX: |
| 82 | + raise ValueError("head_dim must be a positive TensorRT dimension") |
| 83 | + return head_dim |
| 84 | + |
| 85 | + |
| 86 | +def _checked_product(label: str, *values: int) -> int: |
| 87 | + product = 1 |
| 88 | + for value in values: |
| 89 | + if value <= 0 or product > _UINT64_MAX // value: |
| 90 | + raise ValueError(f"native Qwen KV {label} exceeds uint64") |
| 91 | + product *= value |
| 92 | + return product |
| 93 | + |
| 94 | + |
| 95 | +def native_kv_cache_geometry( |
| 96 | + config: object, |
| 97 | + capacity: int, |
| 98 | + *, |
| 99 | + element_bytes: int = 2, |
| 100 | +) -> tuple[int, int]: |
| 101 | + """Return runtime byte geometry for one fixed native cache capacity.""" |
| 102 | + |
| 103 | + capacity = _integer(capacity, "max_cache_length") |
| 104 | + context = _positive(config, "max_position_embeddings") |
| 105 | + if capacity <= 0 or capacity > context: |
| 106 | + raise ValueError( |
| 107 | + "native Qwen KV requires max_cache_length in " |
| 108 | + f"[1, max_position_embeddings ({context})], got {capacity}" |
| 109 | + ) |
| 110 | + row_bytes = _checked_product( |
| 111 | + "row size", |
| 112 | + 2, |
| 113 | + _positive(config, "num_hidden_layers"), |
| 114 | + _positive(config, "num_key_value_heads"), |
| 115 | + resolved_head_dim(config), |
| 116 | + _integer(element_bytes, "element_bytes"), |
| 117 | + ) |
| 118 | + return row_bytes, _checked_product("cache size", capacity, row_bytes) |
| 119 | + |
| 120 | + |
| 121 | +def _enabled(value: object) -> bool: |
| 122 | + return value not in (None, False, 0, "", (), [], {}) |
| 123 | + |
| 124 | + |
| 125 | +def _validate_default_rope(raw: dict, reasons: list[str]) -> None: |
| 126 | + parameters = raw.get("rope_parameters") |
| 127 | + scaling = raw.get("rope_scaling") |
| 128 | + if parameters is not None and scaling is not None: |
| 129 | + reasons.append("RoPE configuration is ambiguous") |
| 130 | + return |
| 131 | + rope = parameters if parameters is not None else scaling |
| 132 | + if rope is None: |
| 133 | + return |
| 134 | + if not isinstance(rope, dict): |
| 135 | + reasons.append("RoPE configuration must be an object") |
| 136 | + return |
| 137 | + rope_type = str(rope.get("rope_type", rope.get("type", "default"))).lower() |
| 138 | + if rope_type not in ("", "default") or any( |
| 139 | + key in rope |
| 140 | + for key in ( |
| 141 | + "attention_factor", |
| 142 | + "beta_fast", |
| 143 | + "beta_slow", |
| 144 | + "factor", |
| 145 | + "original_max_position_embeddings", |
| 146 | + ) |
| 147 | + ): |
| 148 | + reasons.append("native Qwen3 supports only unscaled default RoPE") |
| 149 | + |
| 150 | + |
| 151 | +def native_kv_architecture_capability( |
| 152 | + config: object, |
| 153 | +) -> NativeKvCapability: |
| 154 | + """Accept any model size that retains the dense Qwen3 graph contract.""" |
| 155 | + |
| 156 | + if str(getattr(config, "model_type", "")).lower() != "qwen3": |
| 157 | + return _result(applicable=False) |
| 158 | + |
| 159 | + raw = _raw(config) |
| 160 | + reasons: list[str] = [] |
| 161 | + if tuple(getattr(config, "architectures", ()) or ()) != ("Qwen3ForCausalLM",): |
| 162 | + reasons.append("architectures must contain exactly Qwen3ForCausalLM") |
| 163 | + |
| 164 | + try: |
| 165 | + dimensions = { |
| 166 | + name: _positive(config, name) |
| 167 | + for name in ( |
| 168 | + "vocab_size", |
| 169 | + "hidden_size", |
| 170 | + "intermediate_size", |
| 171 | + "num_hidden_layers", |
| 172 | + "num_attention_heads", |
| 173 | + "num_key_value_heads", |
| 174 | + "max_position_embeddings", |
| 175 | + ) |
| 176 | + } |
| 177 | + head_dim = resolved_head_dim(config) |
| 178 | + if dimensions["num_attention_heads"] % dimensions["num_key_value_heads"]: |
| 179 | + reasons.append("num_attention_heads must be divisible by num_key_value_heads") |
| 180 | + if head_dim != 128: |
| 181 | + reasons.append("native Qwen3 attention requires head_dim=128") |
| 182 | + except ValueError as exc: |
| 183 | + reasons.append(str(exc)) |
| 184 | + |
| 185 | + if str(getattr(config, "hidden_act", "")).lower() != "silu": |
| 186 | + reasons.append("native Qwen3 requires hidden_act='silu'") |
| 187 | + for name in ("rms_norm_eps", "rope_theta"): |
| 188 | + try: |
| 189 | + value = float(getattr(config, name)) |
| 190 | + except (TypeError, ValueError, OverflowError): |
| 191 | + value = 0.0 |
| 192 | + if not math.isfinite(value) or value <= 0: |
| 193 | + reasons.append(f"{name} must be finite and positive") |
| 194 | + |
| 195 | + unsupported_flags = ( |
| 196 | + "attention_bias", |
| 197 | + "mlp_bias", |
| 198 | + "is_encoder_decoder", |
| 199 | + "use_sliding_window", |
| 200 | + "sliding_window", |
| 201 | + "rope_interleaved", |
| 202 | + "interleaved_rope", |
| 203 | + "num_experts", |
| 204 | + "num_local_experts", |
| 205 | + "num_experts_per_tok", |
| 206 | + "moe_intermediate_size", |
| 207 | + "shared_expert_intermediate_size", |
| 208 | + "full_attention_interval", |
| 209 | + "linear_conv_kernel_dim", |
| 210 | + "linear_key_head_dim", |
| 211 | + "linear_num_key_heads", |
| 212 | + "linear_num_value_heads", |
| 213 | + "linear_value_head_dim", |
| 214 | + ) |
| 215 | + enabled = [name for name in unsupported_flags if _enabled(raw.get(name))] |
| 216 | + if enabled: |
| 217 | + reasons.append("unsupported Qwen3 fields: " + ", ".join(enabled)) |
| 218 | + |
| 219 | + try: |
| 220 | + if float(raw.get("partial_rotary_factor", 1.0)) != 1.0: |
| 221 | + reasons.append("native Qwen3 requires full rotary embeddings") |
| 222 | + except (TypeError, ValueError, OverflowError): |
| 223 | + reasons.append("partial_rotary_factor must be numeric") |
| 224 | + layer_types = raw.get("layer_types") |
| 225 | + if layer_types is not None and ( |
| 226 | + not isinstance(layer_types, (list, tuple)) |
| 227 | + or any(str(value).lower() != "full_attention" for value in layer_types) |
| 228 | + ): |
| 229 | + reasons.append("native Qwen3 does not support hybrid layer types") |
| 230 | + _validate_default_rope(raw, reasons) |
| 231 | + return _result(reasons=reasons) |
| 232 | + |
| 233 | + |
| 234 | +def native_kv_build_capability( |
| 235 | + config: object, |
| 236 | + *, |
| 237 | + precision: str = "bf16", |
| 238 | + max_cache_length: int | None = None, |
| 239 | + parallel_enabled: bool | None = None, |
| 240 | + quantized: bool | None = None, |
| 241 | + debug_layer_outputs: bool = False, |
| 242 | +) -> NativeKvCapability: |
| 243 | + """Apply deployment constraints once, after architecture routing.""" |
| 244 | + |
| 245 | + architecture = native_kv_architecture_capability(config) |
| 246 | + if not architecture.eligible: |
| 247 | + return architecture |
| 248 | + |
| 249 | + raw = _raw(config) |
| 250 | + reasons: list[str] = [] |
| 251 | + if str(precision).lower() not in {"fp16", "bf16"}: |
| 252 | + reasons.append("native Qwen3 requires FP16 or BF16") |
| 253 | + if str(raw.get("_decoder_engine_layout", "split")) != "split": |
| 254 | + reasons.append("native Qwen3 requires split prefill/decode engines") |
| 255 | + if parallel_enabled or raw.get("_parallel_build_enabled"): |
| 256 | + reasons.append("native Qwen3 does not support tensor parallel builds") |
| 257 | + if quantized or raw.get("quantization_config") or raw.get("_quantized_build_requested"): |
| 258 | + reasons.append("native Qwen3 does not support quantized builds") |
| 259 | + if raw.get("_fp32_layers"): |
| 260 | + reasons.append("native Qwen3 does not support FP32 layer overrides") |
| 261 | + if debug_layer_outputs: |
| 262 | + reasons.append("native Qwen3 does not support debug layer outputs") |
| 263 | + try: |
| 264 | + native_kv_cache_geometry( |
| 265 | + config, |
| 266 | + ( |
| 267 | + int(getattr(config, "max_position_embeddings")) |
| 268 | + if max_cache_length is None |
| 269 | + else max_cache_length |
| 270 | + ), |
| 271 | + ) |
| 272 | + except ValueError as exc: |
| 273 | + reasons.append(str(exc)) |
| 274 | + return _result(reasons=reasons) |
0 commit comments