From 252a166f122329dec80a03c165120a59a11f2aa2 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Thu, 21 May 2026 15:13:18 +0200 Subject: [PATCH] sort based on abs tkv difference Signed-off-by: Yannick Schnider --- sendnn_inference/v1/core/scheduler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sendnn_inference/v1/core/scheduler.py b/sendnn_inference/v1/core/scheduler.py index f07b5c878..a0d4979f9 100644 --- a/sendnn_inference/v1/core/scheduler.py +++ b/sendnn_inference/v1/core/scheduler.py @@ -273,6 +273,13 @@ def schedule(self) -> "SchedulerOutput": while self.skipped_waiting: holdback_queue.append(self.skipped_waiting.pop_request()) + # Greedily pick the request whose prompt length is closest to the + # current batch TKV so the incoming sequence aligns with the running + # decode batch as tightly as possible. May starve outliers — intentional. + holdback_queue = deque( + sorted(holdback_queue, key=lambda r: abs(r.num_prompt_tokens - self.tkv)) + ) + # Check if new requests can be scheduled for prefill while holdback_queue: if self.can_schedule_prefill(holdback_queue[0]):