diff --git a/sendnn_inference/v1/core/scheduler.py b/sendnn_inference/v1/core/scheduler.py index 9dc742d4c..860680d4b 100644 --- a/sendnn_inference/v1/core/scheduler.py +++ b/sendnn_inference/v1/core/scheduler.py @@ -273,6 +273,11 @@ def schedule(self) -> "SchedulerOutput": while self.skipped_waiting: holdback_queue.append(self.skipped_waiting.pop_request()) + # Sort by prompt length ascending so shorter requests are scheduled + # first, maximising homogeneous-batch opportunities (may starve long + # requests indefinitely — intentional for experimentation). + holdback_queue = deque(sorted(holdback_queue, key=lambda r: r.num_prompt_tokens)) + # Check if new requests can be scheduled for prefill while holdback_queue: if self.can_schedule_prefill(holdback_queue[0]):