add only whoe image match

Author: RunningLeon

lmdeploy/api.py

@@ -76,11 +76,6 @@ def pipeline(model_path: str,
     backend = 'pytorch' if type(backend_config) is PytorchEngineConfig else 'turbomind'
     logger.info(f'Using {backend} engine')
 
-    if task == 'vlm':
-        if backend_config and backend_config.enable_prefix_caching and backend == 'turbomind':
-            backend_config.enable_prefix_caching = False
-            logger.warning('VLM does not support prefix caching for turbomind engine.')
-
     return pipeline_class(model_path,
                           backend=backend,
                           backend_config=backend_config,

lmdeploy/pytorch/messages.py

@@ -371,47 +371,66 @@ def __init__(self, multimodals: MultiModalInputs):
         if multimodals is None:
             multimodals = dict()
         self.multimodals = multimodals
+        self._init_mm_ranges()
+
+    def _init_mm_ranges(self):
+        """init mm ranges and sort it."""
+        mm_ranges = []
+        for _, modal_datas in self.multimodals.items():
+            for modal_data in modal_datas:
+                data = (modal_data.start, modal_data.end, modal_data.meta.get('hash_value', None))
+                mm_ranges.append(data)
+        mm_ranges.sort(key=lambda x: x[1])
+        self._mm_ranges = mm_ranges
+
+    @property
+    def mm_ranges(self):
+        """mm_ranges"""
+        return self._mm_ranges
 
     def get_datas(self, start=0, end=-1):
         """get multimodals from prompts position [start, end)."""
         outs = dict()
-        test_range = range(start, end)
         for modal_type, modal_datas in self.multimodals.items():
             data = []
             for modal_data in modal_datas:
-                if (modal_data.start not in test_range and modal_data.end not in test_range):
-                    continue
-                data.append(modal_data)
+                if modal_data.start < end and modal_data.end > start:
+                    data.append(modal_data)
             if len(data) > 0:
                 outs[modal_type] = data
         return outs
 
-    def get_step(self, step: int):
+    def get_step(self, step: int) -> int:
         """get step that before a whole image."""
         real_step = step
-        for modal_type, modal_datas in self.multimodals.items():
-            for modal_data in modal_datas:
-                if modal_data.start > real_step:
-                    continue
-                elif modal_data.end <= real_step:
-                    continue
-                else:
-                    real_step = modal_data.start
+        for start, end, _ in self._mm_ranges:
+            if start <= real_step < end:
+                real_step = start
         return real_step
 
+    def has_data(self, start: int, end: int) -> bool:
+        """whether has multimodal data in [start, end)"""
+        return any([s < end and e > start for s, e, _ in self._mm_ranges])
+
     def get_hash_values(self, start: int, end: int):
         """get multimodals hash values that from [start, end)"""
-        hash_values = []
-        for modal_type, modal_datas in self.multimodals.items():
-            for modal_data in modal_datas:
-                if modal_data.start < end and modal_data.end > start:
-                    if modal_data.meta.get('hash_value', None):
-                        hash_values.append(modal_data.meta['hash_value'])
-        if hash_values:
-            hash_values = tuple(hash_values)
+        mm_hash_values = []
+        multimodal_ends = []
+
+        for mm_start, mm_end, hash_value in self._mm_ranges:
+            # the mm range intersect with the target range
+            if mm_start < end and mm_end > start:
+                mm_hash_values.append(hash_value)
+                # the mm end in the target range
+                if start < mm_end <= end:
+                    cur_data = (tuple(mm_hash_values), mm_end)
+                    multimodal_ends.append(cur_data)
+
+        if len(mm_hash_values) == 0:
+            mm_hash_values = None
         else:
-            hash_values = None
-        return hash_values
+            mm_hash_values = tuple(mm_hash_values)
+        return mm_hash_values, multimodal_ends
 
     def add_inputs(self, input_mms: MultiModalInputs):
         """add new inputs."""
@@ -421,9 +440,17 @@ def add_inputs(self, input_mms: MultiModalInputs):
             else:
                 self.multimodals[modal_type] = vals
 
-    def empty(self):
+            # update mm_ranges
+            for modal_data in vals:
+                data = (modal_data.start, modal_data.end, modal_data.meta.get('hash_value', None))
+                self._mm_ranges.append(data)
+
+        # sort mm_ranges
+        self._mm_ranges.sort(key=lambda x: x[1])
+
+    def empty(self) -> bool:
         if len(self.multimodals) == 0:
-            return 0
+            return True
 
         return all(len(vals) == 0 for vals in self.multimodals)
 
@@ -609,7 +636,7 @@ def update_token_ids(self,
 
         # update multimodals
         if multimodals is not None:
-            multimodals = HistoryMultiModals.update_multimodals(multimodals, self.num_all_ids)
+            multimodals = HistoryMultiModals.update_multimodals(multimodals, self._num_history_ids)
             self.history_multimodals.add_inputs(multimodals)
 
         # cross
@@ -641,6 +668,7 @@ def set_step(self, step: int):
             new_step = self.history_multimodals.get_step(step)
             assert 0 <= new_step <= step
             step = new_step
+
         self._num_history_ids = step
         self._num_token_ids = num_all_ids - step
         self.num_ignored_history = min(step, self.num_ignored_history)
@@ -651,3 +679,16 @@ def set_step(self, step: int):
         if self.history_multimodals is not None:
             self._num_history_cross = self.history_multimodals.get_encoder_len(0, self.num_history_ids)
             self._num_cross = self.history_multimodals.get_encoder_len(self._num_history_ids, num_all_ids)
+
+    def __repr__(self):
+        return (
+                f'SchedulerSequence(seq_id={self.seq_id}, session_id={self.session_id}, '
+                f'status={self.status}, arrive_time={self.arrive_time}, '
+                f'return_logits={self.return_logits}, sampling_param={self.sampling_param}, '
+                f'num_history_tokens={self.history_len}, num_all_tokens={self.num_all_ids}, '
+                f'num_new_tokens={self.num_new_tokens}, all_token_ids={self.all_ids}, '
+                f'num_gpu_blocks={self.num_blocks}, gpu_blocks={self.logical_blocks.get_real_blocks()}, '
+                f'last_shared_node={getattr(self, "last_shared_node", None)})'
+                )
+
+    __str__ = __repr__