From a27077e76b1dd6d7aef8e926680751d2404fe5bf Mon Sep 17 00:00:00 2001 From: Shiv Kaul Date: Wed, 21 Jan 2026 19:36:30 +0000 Subject: [PATCH 01/16] first commit Signed-off-by: Shiv Kaul --- vllm_gaudi/models/interfaces.py | 3 +- vllm_gaudi/models/qwen3_vl.py | 89 +++++++++++++- vllm_gaudi/models/utils.py | 11 +- vllm_gaudi/v1/worker/hpu_model_runner.py | 142 +++++++++++++---------- 4 files changed, 168 insertions(+), 77 deletions(-) diff --git a/vllm_gaudi/models/interfaces.py b/vllm_gaudi/models/interfaces.py index a4ea2327e..26ead7f83 100644 --- a/vllm_gaudi/models/interfaces.py +++ b/vllm_gaudi/models/interfaces.py @@ -1,7 +1,6 @@ from collections.abc import Callable import torch from torch import Tensor -from vllm.model_executor.models.interfaces import SupportsMultiModal def _embed_text_input_ids( @@ -38,4 +37,4 @@ def _embed_text_input_ids( return embed_input_ids(input_ids) -SupportsMultiModal._embed_text_input_ids = _embed_text_input_ids +#SupportsMultiModal._embed_text_input_ids = _embed_text_input_ids diff --git a/vllm_gaudi/models/qwen3_vl.py b/vllm_gaudi/models/qwen3_vl.py index 9dd551155..82a72d3eb 100644 --- a/vllm_gaudi/models/qwen3_vl.py +++ b/vllm_gaudi/models/qwen3_vl.py @@ -1,6 +1,9 @@ -from torch import nn -from vllm.model_executor.layers.activation import get_act_fn +import torch +from .utils import _merge_multimodal_embeddings from vllm.config import VllmConfig +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.models.interfaces import MultiModalEmbeddings +from vllm.model_executor.models.interfaces import _require_is_multimodal from vllm.model_executor.models.qwen3_vl import ( Qwen3VLForConditionalGeneration, Qwen3_VisionTransformer, @@ -65,9 +68,9 @@ def __init__( ) depth = vision_config.depth - norm_layer = lambda d: nn.LayerNorm(d, eps=norm_eps) + norm_layer = lambda d: torch.nn.LayerNorm(d, eps=norm_eps) - self.blocks = nn.ModuleList([ + self.blocks = torch.nn.ModuleList([ HPUQwen3_VisionBlock( dim=self.hidden_size, num_heads=self.num_heads, @@ -97,3 +100,81 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): multimodal_config=multimodal_config, prefix=maybe_prefix(prefix, "visual"), ) + + def _compute_deepstack_embeds( + self, + inputs_embeds: torch.Tensor, + multimodal_embeddings: MultiModalEmbeddings, + is_multimodal: torch.Tensor, + ) -> tuple[torch.Tensor, MultiModalEmbeddings]: + visual_lens = [len(x) for x in multimodal_embeddings] + multimodal_embeddings_cat = torch.cat(multimodal_embeddings, dim=0) + + ( + multimodal_embeddings_main, + multimodal_embeddings_multiscale, + ) = torch.split( + multimodal_embeddings_cat, + [self.visual_dim, self.multiscale_dim], + dim=-1, + ) + + multimodal_embeddings = torch.split(multimodal_embeddings_main, visual_lens, dim=0) + multimodal_embeddings_multiscale = torch.split(multimodal_embeddings_multiscale, visual_lens, dim=0) + + deepstack_input_embeds = inputs_embeds.new_zeros(inputs_embeds.size(0), + self.deepstack_num_level * inputs_embeds.size(1)) + + deepstack_input_embeds = _merge_multimodal_embeddings( + inputs_embeds=deepstack_input_embeds, + multimodal_embeddings=multimodal_embeddings_multiscale, + is_multimodal=is_multimodal, + ) + deepstack_input_embeds = deepstack_input_embeds.view(inputs_embeds.shape[0], self.deepstack_num_level, + self.visual_dim) + deepstack_input_embeds = deepstack_input_embeds.permute(1, 0, 2) + + return deepstack_input_embeds, multimodal_embeddings + + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: MultiModalEmbeddings | None = None, + *, + is_multimodal: torch.Tensor | None = None, + handle_oov_mm_token: bool = False, + ) -> torch.Tensor: + inputs_embeds = self._embed_text_input_ids( + input_ids, + self.language_model.embed_input_ids, + is_multimodal=is_multimodal, + handle_oov_mm_token=handle_oov_mm_token, + ) + + if multimodal_embeddings is None or len(multimodal_embeddings) == 0: + return inputs_embeds + + is_multimodal = _require_is_multimodal(is_multimodal) + + if self.use_deepstack: + ( + deepstack_input_embeds, + multimodal_embeddings, + ) = self._compute_deepstack_embeds( + inputs_embeds=inputs_embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + else: + deepstack_input_embeds = None + + inputs_embeds = _merge_multimodal_embeddings( + inputs_embeds=inputs_embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + + if deepstack_input_embeds is not None: + self._set_deepstack_input_embeds(deepstack_input_embeds) + + return inputs_embeds diff --git a/vllm_gaudi/models/utils.py b/vllm_gaudi/models/utils.py index c1bbfd0e4..e7c090040 100644 --- a/vllm_gaudi/models/utils.py +++ b/vllm_gaudi/models/utils.py @@ -29,18 +29,15 @@ def _merge_multimodal_embeddings( mm_embeds_flat = _flatten_embeddings(multimodal_embeddings) input_dtype = inputs_embeds.dtype + if is_multimodal.dtype == torch.int64: + return inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) try: # For debugging # inputs_embeds[is_multimodal] = mm_embeds_flat.to(dtype=input_dtype) - # htcore.mark_step() + # NOTE: This can avoid D2H sync (#22105), but fails to # raise an error if is_multimodal.sum() < len(mm_embeds_flat) - # inputs_embeds.masked_scatter_(is_multimodal.unsqueeze(-1), - # mm_embeds_flat.to(dtype=input_dtype)) - - multimodal_positions = torch.where(is_multimodal)[0][:mm_embeds_flat.shape[0]] - inputs_embeds[0, multimodal_positions] = mm_embeds_flat.to(dtype=input_dtype) - + inputs_embeds.masked_scatter_(is_multimodal.unsqueeze(-1), mm_embeds_flat.to(dtype=input_dtype)) except RuntimeError as e: num_actual_tokens = len(mm_embeds_flat) num_expected_tokens = is_multimodal.sum().item() diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 541334cd9..8f95c7420 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 import collections +import copy import contextlib import functools from functools import partial @@ -48,7 +49,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.vocab_parallel_embedding import (VocabParallelEmbedding) from vllm.model_executor.model_loader import get_model, get_model_loader -from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.multimodal.inputs import (BatchedTensorInputs, MultiModalKwargsItem) from vllm.multimodal.utils import group_mm_kwargs_by_modality from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding @@ -75,7 +76,7 @@ from vllm.model_executor.models.interfaces_base import (VllmModelForPooling, is_pooling_model, is_text_generation_model) from vllm.tasks import GenerationTask, PoolingTask, SupportedTask from vllm.transformers_utils.config import is_interleaved -from vllm.v1.worker.utils import (gather_mm_placeholders, sanity_check_mm_encoder_outputs, scatter_mm_placeholders) +from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs from vllm.v1.sample.rejection_sampler import RejectionSampler from vllm.v1.spec_decode.eagle import EagleProposer from vllm.v1.spec_decode.metadata import SpecDecodeMetadata @@ -691,7 +692,7 @@ def __init__( self.head_size = self.model_config.get_head_size() self.hidden_size = self.model_config.get_hidden_size() self.is_pooling_model = (model_config.runner_type == 'pooling') - logger.debug("model config: ", self.model_config) + logger.debug("model config: %s", self.model_config) self.attn_backend = get_attn_backend( self.head_size, @@ -704,10 +705,12 @@ def __init__( # Mult-modal-related. self.mm_registry = MULTIMODAL_REGISTRY self.uses_mrope = model_config.uses_mrope + self.model_config_copy = None self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(model_config) if self.supports_mm_inputs: self.is_mm_embed = self._make_buffer(self.max_num_tokens, dtype=torch.bool) + self.model_config_copy = copy.deepcopy(self.model_config) self.is_multimodal_raw_input_supported = (model_config.is_multimodal_raw_input_only_model) # Lazy initialization @@ -1265,11 +1268,7 @@ def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput", req_ids: list if req_id not in self.encoder_cache: self.encoder_cache[req_id] = {} - self.encoder_cache[mm_hash] = scatter_mm_placeholders( - output, - is_embed=pos_info.is_embed.to( - device=output.device) if pos_info.is_embed is not None else pos_info.is_embed, - ) + self.encoder_cache[mm_hash] = output # modified from: vllm/v1/worker/gpu_model_runner.py def _gather_mm_embeddings( @@ -1287,6 +1286,7 @@ def _gather_mm_embeddings( req_start_idx = 0 for req_id in req_ids: + mm_embeds_req: list[torch.Tensor] = [] num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] req_state = self.requests[req_id] num_computed_tokens = \ @@ -1311,6 +1311,11 @@ def _gather_mm_embeddings( start_idx = max(num_computed_tokens - start_pos, 0) end_idx = min(num_computed_tokens - start_pos + num_scheduled_tokens, num_encoder_tokens) assert start_idx < end_idx + curr_embeds_start, curr_embeds_end = (pos_info.get_embeds_indices_in_range(start_idx, end_idx)) + # If there are no embeddings in the current range, we skip + # gathering the embeddings. + if curr_embeds_start == curr_embeds_end: + continue mm_hash = mm_feature.identifier encoder_output = self.encoder_cache.get(mm_hash, None) assert encoder_output is not None,\ @@ -1319,21 +1324,26 @@ def _gather_mm_embeddings( if (is_embed := pos_info.is_embed) is not None: is_embed = is_embed[start_idx:end_idx] + mm_embeds_item = encoder_output[curr_embeds_start:curr_embeds_end] + else: + mm_embeds_item = encoder_output[start_idx:end_idx] - mm_embeds_item = gather_mm_placeholders( - encoder_output[start_idx:end_idx], - is_embed=is_embed, - ) req_start_pos = req_start_idx + start_pos - num_computed_tokens - is_mm_embed[req_start_pos+start_idx:req_start_pos + end_idx] \ - = True - - # Only whole mm items are processed - mm_embeds.append(mm_embeds_item) + is_mm_embed[req_start_pos + start_idx:req_start_pos + + end_idx] = (True if is_embed is None else is_embed) + mm_embeds_req.append(mm_embeds_item) + mm_embeds.extend(mm_embeds_req) req_start_idx += num_scheduled_tokens - is_mm_embed = self.is_mm_embed.copy_to_gpu(total_num_scheduled_tokens) - + # Convert bool tensor to index tensor for merge embedding statically if optimized mm + if self.uses_mrope: + is_mm_embed_index = torch.nonzero(is_mm_embed[:total_num_scheduled_tokens], as_tuple=True)[0] + # Bounds validation on CPU + if len(is_mm_embed_index) > 0 and is_mm_embed_index.max() >= total_num_scheduled_tokens: + raise ValueError(f"Index {is_mm_embed_index.max()} exceeds tensor size {total_num_scheduled_tokens}") + is_mm_embed = is_mm_embed_index.to(self.device) + else: + is_mm_embed = self.is_mm_embed.copy_to_gpu(total_num_scheduled_tokens) return mm_embeds, is_mm_embed def _get_model_mm_inputs( @@ -3752,6 +3762,7 @@ def load_model(self) -> None: self._maybe_compile(self.model) self.model_memory_usage = m.consumed_device_memory logger.info("Compilation took %.4f GB", self.model_memory_usage / float(2**30)) + self.is_mm_optimized = is_mm_optimized(self.model) def _maybe_compile(self, *args, **kwargs): """Entrypoint for a torch.compilation of the model""" @@ -3864,12 +3875,13 @@ def log_warmup(self, phase, i, max_i, first_dim, second_dim, third_dim, causal=F f"free_mem:{free_mem}") tqdm.write(msg) - def log_warmup_multimodal(self, phase, i, max_i, batch_size, seq_len, img_args): + def log_warmup_multimodal(self, phase, i, max_i, batch_size, seq_len, w, h, f): free_mem = format_bytes(HabanaMemoryProfiler.current_free_device_memory()) msg = (f"[Warmup][{phase}][{i+1}/{max_i}] " f"batch_size:{batch_size} " f"seq_len:{seq_len} " - f"img_args:{img_args} " + f"resolution:{w}X{h} " + f"frames:{f} " f"free_mem:{free_mem}") logger.info(msg) @@ -4551,51 +4563,46 @@ def _get_mm_dummy_batch( ) -> BatchedTensorInputs: """Dummy data for profiling and precompiling multimodal models.""" assert self.mm_budget is not None - img_count = 1 - batch = image_args if self.get_model().vision_bucket_manager.is_batch_based else img_count - '''if self.get_model().vision_bucket_manager.is_batch_based: + count = 1 + num_frames = 0 + batch = image_args if self.get_model().vision_bucket_manager.is_batch_based else count + if self.get_model().vision_bucket_manager.is_batch_based: # Create ImageDummyOptions for Gemma3 - #image_options = ImageDummyOptions( - # width=896, # pixels as in gemma3 config - # height=896 # pixels as in gemma3 config - #) + w = 896, # pixels as in gemma3 config + h = 896 # pixels as in gemma3 config batch = image_args else: - #patch_size = int(self.get_patch_size_from_model()) + patch_size = int(self.get_patch_size_from_model()) # Calculate width and height to maintain aspect ratio and patch count # Total patches = (width/patch_size) * (height/patch_size) # We want: (w/ps) * (h/ps) = num_patch where num_patch is image_args # And: w/h = ratio_w/ratio_h - #grid_w = int(math.sqrt(image_args * ratio_w / ratio_h)) - #grid_h = int(image_args / grid_w) - #w = grid_w * patch_size - #h = grid_h * patch_size - #image_options = ImageDummyOptions( - # width=w, # Custom width in pixels - # height=h # Custom height in pixels - #) - batch = img_count - - processor = self.mm_registry.create_processor(model_config=self.model_config, cache=self.mm_budget.cache) - dummy_data = processor.dummy_inputs.get_decoder_dummy_data(processor, - seq_len=4096, - mm_counts={"image": img_count}, - mm_options={"image": image_options}), - - dummy_mm_data = processor.dummy_inputs.get_dummy_processor_inputs( - seq_len=4096, - mm_counts={"image": img_count}, - ) - ''' - - assert modality == 'image' - # Result in the maximum GPU consumption of the model - dummy_mm_inputs = self.mm_registry.get_dummy_mm_inputs( - self.model_config, - mm_counts={modality: 1}, - cache=self.mm_budget.cache, - ) + grid_w = int(math.sqrt(image_args * ratio_w / ratio_h)) + grid_h = int(image_args / grid_w) + w = grid_w * patch_size + h = grid_h * patch_size + batch = count + self.model_config_copy.max_model_len = 4096 + if modality == 'image': + self.model_config_copy.limit_mm_per_prompt = {"image": {"count": count, "width": w, "height": h}} + elif modality == 'video': + video_options = self.model_config_copy.get_multimodal_config().get_dummy_options("video") + num_frames = video_options.num_frames if video_options and hasattr(video_options, 'num_frames') else 100 + w = video_options.width if video_options and hasattr(video_options, 'width') else w + h = video_options.height if video_options and hasattr(video_options, 'height') else h + count = video_options.count if video_options and hasattr(video_options, 'count') else 1 + self.model_config_copy.limit_mm_per_prompt = { + "video": { + "count": count, + "num_frames": num_frames, + "width": w, + "height": h + } + } + else: + raise NotImplementedError(f"Modality '{modality}' is not supported") + dummy_mm_inputs = MultiModalRegistry().get_dummy_mm_inputs(self.model_config_copy, mm_counts={modality: count}) dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0] # We use the cache so that the item is saved to the cache, # but not read from the cache @@ -4607,7 +4614,7 @@ def _get_mm_dummy_batch( dummy_mm_items, device=self.device, pin_memory=self.pin_memory, - )) + )), w, h, num_frames def warmup_multimodal_graphs(self, buckets): @@ -4629,15 +4636,22 @@ def warmup_multimodal_graphs(self, buckets): (9, 16), # 9:16 portrait ] aspect_ratios.extend(aspect_ratio_ext) + is_video_warmup = bool(self.model_config.get_multimodal_config() is not None and \ + self.model_config.get_multimodal_config().get_dummy_options("video") is not None \ + and self.mm_budget.mm_limits['video'] != 999) + + is_image_warmup = bool(self.model_config.get_multimodal_config() is not None and \ + self.model_config.get_multimodal_config().get_dummy_options("image") is not None \ + and self.mm_budget.mm_limits['image'] != 0) for modality, max_items in self.mm_budget.mm_limits.items(): - if modality == 'video': - logger.warning_once("Warming up for video is not implemented") + if modality == 'image' and not is_image_warmup or modality == 'video' \ + and not is_video_warmup: continue phase = f'Graph/Multimodal({modality})' num_candidates = len(buckets) for idx, img_arg in enumerate(buckets): for (ratio_w, ratio_h) in aspect_ratios: - batched_dummy_mm_inputs = self._get_mm_dummy_batch(modality, img_arg, ratio_w, ratio_h) + batched_dummy_mm_inputs, w, h, f = self._get_mm_dummy_batch(modality, img_arg, ratio_w, ratio_h) dummy_encoder_outputs = \ self.model.embed_multimodal( **batched_dummy_mm_inputs) @@ -4648,7 +4662,7 @@ def warmup_multimodal_graphs(self, buckets): ) self.graphed_buckets.add(img_arg) - self.log_warmup_multimodal(phase, idx, num_candidates, 1, 0, img_arg) + self.log_warmup_multimodal(phase, idx, num_candidates, 1, 0, w, h, f) def _maybe_profile_unified_attn(self): unified_cfg_str = os.environ.get('VLLM_PROFILE_UNIFIED', None) @@ -5452,7 +5466,7 @@ def __init__( if self.interleaved_sliding_window: self.use_window_sdpa = with_default(get_config().PT_HPU_SDPA_QKV_SLICE_MODE_FWD, False) #os.getenv("PT_HPU_SDPA_QKV_SLICE_MODE_FWD", "false").strip().lower() in ("1", "true") - self.slice_size = with_default(get_config().PT_HPU_SDPA_BC_FACTOR, False) + self.slice_size = int(with_default(get_config().PT_HPU_SDPA_BC_FACTOR, "1024")) # int(os.getenv("PT_HPU_SDPA_BC_FACTOR", "1024")) self.slice_thld = int(os.environ.get('VLLM_FUSEDSDPA_SLIDE_THLD', '8192')) From dd9d65c0dc55b04f2e2282197c0046919b901714 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 15:09:25 -0800 Subject: [PATCH 02/16] modified warmup based on 0.14.0 --- vllm_gaudi/v1/worker/hpu_model_runner.py | 37 +++++++++++++++--------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 8f95c7420..91e439b96 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -43,6 +43,7 @@ from vllm.v1.attention.selector import get_attn_backend from vllm.config import (VllmConfig, update_config) +from vllm.config.multimodal import ImageDummyOptions, VideoDummyOptions from vllm.distributed.kv_transfer import (get_kv_transfer_group, has_kv_transfer_group) from vllm.forward_context import set_forward_context from vllm.model_executor.layers.fused_moe.layer import FusedMoE @@ -51,6 +52,7 @@ from vllm.model_executor.model_loader import get_model, get_model_loader from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.multimodal.inputs import (BatchedTensorInputs, MultiModalKwargsItem) +from vllm.multimodal.profiling import MultiModalProfiler from vllm.multimodal.utils import group_mm_kwargs_by_modality from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding from vllm.multimodal.inputs import PlaceholderRange @@ -705,12 +707,10 @@ def __init__( # Mult-modal-related. self.mm_registry = MULTIMODAL_REGISTRY self.uses_mrope = model_config.uses_mrope - self.model_config_copy = None self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(model_config) if self.supports_mm_inputs: self.is_mm_embed = self._make_buffer(self.max_num_tokens, dtype=torch.bool) - self.model_config_copy = copy.deepcopy(self.model_config) self.is_multimodal_raw_input_supported = (model_config.is_multimodal_raw_input_only_model) # Lazy initialization @@ -4582,27 +4582,38 @@ def _get_mm_dummy_batch( w = grid_w * patch_size h = grid_h * patch_size batch = count - self.model_config_copy.max_model_len = 4096 + if modality == 'image': - self.model_config_copy.limit_mm_per_prompt = {"image": {"count": count, "width": w, "height": h}} + mm_options: Mapping[str, BaseDummyOptions] = { + "image": ImageDummyOptions( + count=count, + width=w, + height=h + ), + "video": None + } elif modality == 'video': - video_options = self.model_config_copy.get_multimodal_config().get_dummy_options("video") + video_options = self.model_config.get_multimodal_config().get_dummy_options("video") num_frames = video_options.num_frames if video_options and hasattr(video_options, 'num_frames') else 100 w = video_options.width if video_options and hasattr(video_options, 'width') else w h = video_options.height if video_options and hasattr(video_options, 'height') else h count = video_options.count if video_options and hasattr(video_options, 'count') else 1 - self.model_config_copy.limit_mm_per_prompt = { - "video": { - "count": count, - "num_frames": num_frames, - "width": w, - "height": h - } + mm_options = VideoDummyOptions(width=w, height=h, num_frames=num_frames) + mm_options: Mapping[str, BaseDummyOptions] = { + "image": None, + "video": VideoDummyOptions( + count=count, + num_frames=num_frames, + width=w, + height=h + ) } else: raise NotImplementedError(f"Modality '{modality}' is not supported") - dummy_mm_inputs = MultiModalRegistry().get_dummy_mm_inputs(self.model_config_copy, mm_counts={modality: count}) + processor = MULTIMODAL_REGISTRY.create_processor(self.model_config) + profiler = MultiModalProfiler(processor) + dummy_mm_inputs = profiler._get_dummy_mm_inputs(seq_len=4196, mm_counts={modality: count}, mm_options=mm_options) dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0] # We use the cache so that the item is saved to the cache, # but not read from the cache From 306eaa6b58a74bb8d27e6772737443a8c6d98697 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 15:51:33 -0800 Subject: [PATCH 03/16] fix precommit --- vllm_gaudi/v1/worker/hpu_model_runner.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 91e439b96..3ce3f9c05 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -50,7 +50,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.vocab_parallel_embedding import (VocabParallelEmbedding) from vllm.model_executor.model_loader import get_model, get_model_loader -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry +from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import (BatchedTensorInputs, MultiModalKwargsItem) from vllm.multimodal.profiling import MultiModalProfiler from vllm.multimodal.utils import group_mm_kwargs_by_modality @@ -4585,11 +4585,7 @@ def _get_mm_dummy_batch( if modality == 'image': mm_options: Mapping[str, BaseDummyOptions] = { - "image": ImageDummyOptions( - count=count, - width=w, - height=h - ), + "image": ImageDummyOptions(count=count, width=w,height=h), "video": None } elif modality == 'video': @@ -4601,19 +4597,16 @@ def _get_mm_dummy_batch( mm_options = VideoDummyOptions(width=w, height=h, num_frames=num_frames) mm_options: Mapping[str, BaseDummyOptions] = { "image": None, - "video": VideoDummyOptions( - count=count, - num_frames=num_frames, - width=w, - height=h - ) + "video": VideoDummyOptions(count=count, num_frames=num_frames, width=w, height=h) } else: raise NotImplementedError(f"Modality '{modality}' is not supported") processor = MULTIMODAL_REGISTRY.create_processor(self.model_config) profiler = MultiModalProfiler(processor) - dummy_mm_inputs = profiler._get_dummy_mm_inputs(seq_len=4196, mm_counts={modality: count}, mm_options=mm_options) + dummy_mm_inputs = profiler._get_dummy_mm_inputs(seq_len=4196, + mm_counts={modality: count}, + mm_options=mm_options) dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0] # We use the cache so that the item is saved to the cache, # but not read from the cache From 4fe7b4e6de2021c8a022a2a67db841d093832837 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 16:10:58 -0800 Subject: [PATCH 04/16] fix precommit --- vllm_gaudi/v1/worker/hpu_model_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 3ce3f9c05..371583523 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -4585,7 +4585,7 @@ def _get_mm_dummy_batch( if modality == 'image': mm_options: Mapping[str, BaseDummyOptions] = { - "image": ImageDummyOptions(count=count, width=w,height=h), + "image": ImageDummyOptions(count=count, width=w, height=h), "video": None } elif modality == 'video': From 00169ce2ff0e74b5af5f93823c09791c4c579dc7 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 16:13:30 -0800 Subject: [PATCH 05/16] fix precommit --- vllm_gaudi/v1/worker/hpu_model_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 371583523..a1de7a418 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 import collections -import copy import contextlib import functools from functools import partial From f32417b826101624560dd82f258fc032576eb683 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 16:22:59 -0800 Subject: [PATCH 06/16] fix precommit --- vllm_gaudi/v1/worker/hpu_model_runner.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index a1de7a418..aa8ada8ca 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -11,7 +11,7 @@ from contextlib import suppress from tqdm import tqdm from dataclasses import dataclass, field, fields -from typing import (TYPE_CHECKING, Any, Callable, Optional, TypeAlias, Union, cast) +from typing import (TYPE_CHECKING, Any, Callable, Optional, TypeAlias, Union, cast, Mapping) if os.getenv("QUANT_CONFIG", None) is not None: from neural_compressor.torch.quantization import finalize_calibration else: @@ -4565,9 +4565,10 @@ def _get_mm_dummy_batch( count = 1 num_frames = 0 batch = image_args if self.get_model().vision_bucket_manager.is_batch_based else count + mm_options: Mapping[str, BaseDummyOptions] = None if self.get_model().vision_bucket_manager.is_batch_based: # Create ImageDummyOptions for Gemma3 - w = 896, # pixels as in gemma3 config + w = 896 # pixels as in gemma3 config h = 896 # pixels as in gemma3 config batch = image_args else: @@ -4583,19 +4584,14 @@ def _get_mm_dummy_batch( batch = count if modality == 'image': - mm_options: Mapping[str, BaseDummyOptions] = { - "image": ImageDummyOptions(count=count, width=w, height=h), - "video": None - } + mm_options = {"image": ImageDummyOptions(count=count, width=w, height=h), "video": None} elif modality == 'video': video_options = self.model_config.get_multimodal_config().get_dummy_options("video") num_frames = video_options.num_frames if video_options and hasattr(video_options, 'num_frames') else 100 w = video_options.width if video_options and hasattr(video_options, 'width') else w h = video_options.height if video_options and hasattr(video_options, 'height') else h count = video_options.count if video_options and hasattr(video_options, 'count') else 1 - mm_options = VideoDummyOptions(width=w, height=h, num_frames=num_frames) - mm_options: Mapping[str, BaseDummyOptions] = { - "image": None, + mm_options = {"image": None, "video": VideoDummyOptions(count=count, num_frames=num_frames, width=w, height=h) } else: From 572cb8dc5adfb4c2117f63c92ae176206c4f558d Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 16:32:06 -0800 Subject: [PATCH 07/16] precommit fix --- vllm_gaudi/v1/worker/hpu_model_runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index aa8ada8ca..c2bbbaa53 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -11,7 +11,8 @@ from contextlib import suppress from tqdm import tqdm from dataclasses import dataclass, field, fields -from typing import (TYPE_CHECKING, Any, Callable, Optional, TypeAlias, Union, cast, Mapping) +from typing import (TYPE_CHECKING, Any, Callable, Optional, TypeAlias, Union, cast) +from collections.abc import Mapping if os.getenv("QUANT_CONFIG", None) is not None: from neural_compressor.torch.quantization import finalize_calibration else: @@ -4591,7 +4592,8 @@ def _get_mm_dummy_batch( w = video_options.width if video_options and hasattr(video_options, 'width') else w h = video_options.height if video_options and hasattr(video_options, 'height') else h count = video_options.count if video_options and hasattr(video_options, 'count') else 1 - mm_options = {"image": None, + mm_options = { + "image": None, "video": VideoDummyOptions(count=count, num_frames=num_frames, width=w, height=h) } else: From 780ff3c9a31c177c139925016b6faf50f9ca24fc Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 16:36:29 -0800 Subject: [PATCH 08/16] precommit fix --- vllm_gaudi/v1/worker/hpu_model_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index c2bbbaa53..377bb6d18 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -44,6 +44,7 @@ from vllm.config import (VllmConfig, update_config) from vllm.config.multimodal import ImageDummyOptions, VideoDummyOptions +from vllm.multimodal.inputs import BaseDummyOptions from vllm.distributed.kv_transfer import (get_kv_transfer_group, has_kv_transfer_group) from vllm.forward_context import set_forward_context from vllm.model_executor.layers.fused_moe.layer import FusedMoE @@ -4566,7 +4567,6 @@ def _get_mm_dummy_batch( count = 1 num_frames = 0 batch = image_args if self.get_model().vision_bucket_manager.is_batch_based else count - mm_options: Mapping[str, BaseDummyOptions] = None if self.get_model().vision_bucket_manager.is_batch_based: # Create ImageDummyOptions for Gemma3 w = 896 # pixels as in gemma3 config @@ -4585,14 +4585,14 @@ def _get_mm_dummy_batch( batch = count if modality == 'image': - mm_options = {"image": ImageDummyOptions(count=count, width=w, height=h), "video": None} + mm_options: Mapping[str, BaseDummyOptions] = {"image": ImageDummyOptions(count=count, width=w, height=h), "video": None} elif modality == 'video': video_options = self.model_config.get_multimodal_config().get_dummy_options("video") num_frames = video_options.num_frames if video_options and hasattr(video_options, 'num_frames') else 100 w = video_options.width if video_options and hasattr(video_options, 'width') else w h = video_options.height if video_options and hasattr(video_options, 'height') else h count = video_options.count if video_options and hasattr(video_options, 'count') else 1 - mm_options = { + mm_options: Mapping[str, BaseDummyOptions] = { "image": None, "video": VideoDummyOptions(count=count, num_frames=num_frames, width=w, height=h) } From e321a4de2de82c08b2679434fa1e2c8d1c683e19 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 16:39:10 -0800 Subject: [PATCH 09/16] precommit fix --- vllm_gaudi/v1/worker/hpu_model_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 377bb6d18..2a816f8d5 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -4585,14 +4585,14 @@ def _get_mm_dummy_batch( batch = count if modality == 'image': - mm_options: Mapping[str, BaseDummyOptions] = {"image": ImageDummyOptions(count=count, width=w, height=h), "video": None} + mm_options = {"image": ImageDummyOptions(count=count, width=w, height=h), "video": None} elif modality == 'video': video_options = self.model_config.get_multimodal_config().get_dummy_options("video") num_frames = video_options.num_frames if video_options and hasattr(video_options, 'num_frames') else 100 w = video_options.width if video_options and hasattr(video_options, 'width') else w h = video_options.height if video_options and hasattr(video_options, 'height') else h count = video_options.count if video_options and hasattr(video_options, 'count') else 1 - mm_options: Mapping[str, BaseDummyOptions] = { + mm_options = { "image": None, "video": VideoDummyOptions(count=count, num_frames=num_frames, width=w, height=h) } From 1666ea0cd0099e27e358dfdf6dcb0e62329ae878 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Wed, 21 Jan 2026 16:41:51 -0800 Subject: [PATCH 10/16] precommit fix --- vllm_gaudi/v1/worker/hpu_model_runner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 2a816f8d5..6562d6ca8 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -12,7 +12,6 @@ from tqdm import tqdm from dataclasses import dataclass, field, fields from typing import (TYPE_CHECKING, Any, Callable, Optional, TypeAlias, Union, cast) -from collections.abc import Mapping if os.getenv("QUANT_CONFIG", None) is not None: from neural_compressor.torch.quantization import finalize_calibration else: @@ -44,7 +43,6 @@ from vllm.config import (VllmConfig, update_config) from vllm.config.multimodal import ImageDummyOptions, VideoDummyOptions -from vllm.multimodal.inputs import BaseDummyOptions from vllm.distributed.kv_transfer import (get_kv_transfer_group, has_kv_transfer_group) from vllm.forward_context import set_forward_context from vllm.model_executor.layers.fused_moe.layer import FusedMoE From 6281c232f0c6f8b56ea1521d992693c8387fa704 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Thu, 22 Jan 2026 16:52:43 -0800 Subject: [PATCH 11/16] fix qwen2.5vl unified attn failure --- vllm_gaudi/models/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vllm_gaudi/models/utils.py b/vllm_gaudi/models/utils.py index e7c090040..e931eb155 100644 --- a/vllm_gaudi/models/utils.py +++ b/vllm_gaudi/models/utils.py @@ -30,7 +30,13 @@ def _merge_multimodal_embeddings( input_dtype = inputs_embeds.dtype if is_multimodal.dtype == torch.int64: - return inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) + if inputs_embeds.ndim == 3 and mm_embeds_flat.ndim == 2: + original_shape = inputs_embeds.shape + inputs_embeds = inputs_embeds.view(-1, inputs_embeds.shape[-1]) + result = inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) + return inputs_embeds.view(original_shape) + else: + return inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) try: # For debugging # inputs_embeds[is_multimodal] = mm_embeds_flat.to(dtype=input_dtype) From 983fbb1155c11ce2642e740bf6581e81bc070d2c Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Thu, 22 Jan 2026 17:22:54 -0800 Subject: [PATCH 12/16] precommit fix --- vllm_gaudi/models/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_gaudi/models/utils.py b/vllm_gaudi/models/utils.py index e931eb155..7aeafd50c 100644 --- a/vllm_gaudi/models/utils.py +++ b/vllm_gaudi/models/utils.py @@ -35,7 +35,7 @@ def _merge_multimodal_embeddings( inputs_embeds = inputs_embeds.view(-1, inputs_embeds.shape[-1]) result = inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) return inputs_embeds.view(original_shape) - else: + else: return inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) try: # For debugging From 6394377f4b981866e1c6366309c80a7db72c79f6 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Thu, 22 Jan 2026 17:28:57 -0800 Subject: [PATCH 13/16] precommit fix --- vllm_gaudi/models/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_gaudi/models/utils.py b/vllm_gaudi/models/utils.py index 7aeafd50c..de5e50672 100644 --- a/vllm_gaudi/models/utils.py +++ b/vllm_gaudi/models/utils.py @@ -33,7 +33,7 @@ def _merge_multimodal_embeddings( if inputs_embeds.ndim == 3 and mm_embeds_flat.ndim == 2: original_shape = inputs_embeds.shape inputs_embeds = inputs_embeds.view(-1, inputs_embeds.shape[-1]) - result = inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) + inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) return inputs_embeds.view(original_shape) else: return inputs_embeds.index_copy_(0, is_multimodal, mm_embeds_flat) From a91323e2cc77af4e66a903db4bbfb2ada1b3d8f1 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Thu, 22 Jan 2026 23:30:15 -0800 Subject: [PATCH 14/16] add more bucket warmup for mm --- vllm_gaudi/extension/bucketing/vision.py | 3 +-- vllm_gaudi/v1/worker/hpu_model_runner.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/vllm_gaudi/extension/bucketing/vision.py b/vllm_gaudi/extension/bucketing/vision.py index 78037de45..ec5ba116a 100644 --- a/vllm_gaudi/extension/bucketing/vision.py +++ b/vllm_gaudi/extension/bucketing/vision.py @@ -21,8 +21,7 @@ }, 'qwen3_vl': { 'is_batch_based': False, - #coverage for lmarena-ai/VisionArena-Chat - 'buckets': [512, 1024, 2048, 3072, 4096, 5120, 6144, 7168, 8192, 9216, 10240, 11264, 12288, 131076] + 'buckets': [256, 512, 1024, 1350, 1602, 2048, 3072, 4096, 5120, 6144, 7168, 8192, 9216, 10240, 11264, 12288, 131076] } } diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 6562d6ca8..100951bda 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -4624,17 +4624,17 @@ def warmup_multimodal_graphs(self, buckets): self.scheduler_config, self.mm_registry, ) if self.supports_mm_inputs else None - aspect_ratios = [(1, 1)] # 1:1 square - sanity_check = False - if self.get_model().vision_bucket_manager.is_batch_based: - sanity_check = True - aspect_ratio_ext = [ - (4, 3), # 4:3 landscape - (3, 4), # 3:4 portrait - (16, 9), # 16:9 widescreen - (9, 16), # 9:16 portrait - ] - aspect_ratios.extend(aspect_ratio_ext) + + sanity_check = True if self.get_model().vision_bucket_manager.is_batch_based else False + + aspect_ratios = [ + (1, 1), # 1:1 square + (4, 3), # 4:3 landscape + (3, 4), # 3:4 portrait + (16, 9), # 16:9 widescreen + (9, 16), # 9:16 portrait + ] + is_video_warmup = bool(self.model_config.get_multimodal_config() is not None and \ self.model_config.get_multimodal_config().get_dummy_options("video") is not None \ and self.mm_budget.mm_limits['video'] != 999) From 51a8d2b1d04e8d4f0c922de706794d4ba5d412e4 Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Thu, 22 Jan 2026 23:44:10 -0800 Subject: [PATCH 15/16] fix precommit --- vllm_gaudi/extension/bucketing/vision.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm_gaudi/extension/bucketing/vision.py b/vllm_gaudi/extension/bucketing/vision.py index ec5ba116a..084526259 100644 --- a/vllm_gaudi/extension/bucketing/vision.py +++ b/vllm_gaudi/extension/bucketing/vision.py @@ -21,7 +21,8 @@ }, 'qwen3_vl': { 'is_batch_based': False, - 'buckets': [256, 512, 1024, 1350, 1602, 2048, 3072, 4096, 5120, 6144, 7168, 8192, 9216, 10240, 11264, 12288, 131076] + 'buckets': + [256, 512, 1024, 1350, 1602, 2048, 3072, 4096, 5120, 6144, 7168, 8192, 9216, 10240, 11264, 12288, 131076] } } From 773b606567a8eeca60b3eebeb54a32812ba8134c Mon Sep 17 00:00:00 2001 From: Libin Tang Date: Thu, 22 Jan 2026 23:47:10 -0800 Subject: [PATCH 16/16] fix precommit --- vllm_gaudi/v1/worker/hpu_model_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_gaudi/v1/worker/hpu_model_runner.py b/vllm_gaudi/v1/worker/hpu_model_runner.py index 100951bda..08e70e329 100644 --- a/vllm_gaudi/v1/worker/hpu_model_runner.py +++ b/vllm_gaudi/v1/worker/hpu_model_runner.py @@ -4625,7 +4625,7 @@ def warmup_multimodal_graphs(self, buckets): self.mm_registry, ) if self.supports_mm_inputs else None - sanity_check = True if self.get_model().vision_bucket_manager.is_batch_based else False + sanity_check = self.get_model().vision_bucket_manager.is_batch_based aspect_ratios = [ (1, 1), # 1:1 square