From 4c03e2d1777d4ed35a6acdd48bbb1349f6f67921 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 30 Sep 2025 21:03:23 +0000 Subject: [PATCH 1/2] fix defaults Signed-off-by: Lucas Wilkinson --- vllm/model_executor/models/config.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 589ca0069034..9392f2270828 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -400,7 +400,7 @@ def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: "exactly equal.", mamba_padding_pct) -class DeepseekV3ForCausalLM(VerifyAndUpdateConfig): +class DeepseekV32ForCausalLM(VerifyAndUpdateConfig): @classmethod def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: @@ -410,19 +410,18 @@ def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: hf_config = vllm_config.model_config.hf_config is_v32 = hasattr(hf_config, "index_topk") + assert is_v32 - if is_v32: - # For DeepSeekV3.2, we use a custom fp8 format as default (i.e. - # "auto") - cache_config = vllm_config.cache_config - if cache_config.cache_dtype == "auto" or \ - cache_config.cache_dtype.startswith("fp8"): - cache_config.cache_dtype = "fp8_ds_mla" - logger.info( - "Using custom fp8 kv-cache format for DeepSeekV3.2") - if cache_config.cache_dtype == "bfloat16": - cache_config.cache_dtype = "auto" - logger.info("Using bfloat16 kv-cache for DeepSeekV3.2") + # For DeepSeekV3.2, we use a custom fp8 format as default (i.e. + # "auto") + cache_config = vllm_config.cache_config + if cache_config.cache_dtype == "auto" or \ + cache_config.cache_dtype.startswith("fp8"): + cache_config.cache_dtype = "fp8_ds_mla" + logger.info("Using custom fp8 kv-cache format for DeepSeekV3.2") + if cache_config.cache_dtype == "bfloat16": + cache_config.cache_dtype = "auto" + logger.info("Using bfloat16 kv-cache for DeepSeekV3.2") MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = { @@ -441,5 +440,5 @@ def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: "MambaForCausalLM": MambaModelConfig, "Mamba2ForCausalLM": MambaModelConfig, "FalconMambaForCausalLM": MambaModelConfig, - "DeepseekV3ForCausalLM": DeepseekV3ForCausalLM, + "DeepseekV32ForCausalLM": DeepseekV32ForCausalLM, } From d67bee8e897a32d52f91b7e5e44354e100f000a3 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 30 Sep 2025 21:08:46 +0000 Subject: [PATCH 2/2] add comment Signed-off-by: Lucas Wilkinson --- vllm/model_executor/models/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 9392f2270828..5711b5ebe85e 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -409,6 +409,7 @@ def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: """ hf_config = vllm_config.model_config.hf_config + # Mirror the check in vllm/model_executor/models/deepseek_v2.py is_v32 = hasattr(hf_config, "index_topk") assert is_v32