Skip to content

Commit 80d4963

Browse files
authored
training: a time-sliced trainer's turn takes in requests that queue during it (#437)
A client sends optim_step right after its forward_backward returns, so the optim_step usually lands while the turn is still running or a moment after it ends. It then waited for a turn of its own: a full wait in the time-slicer's queue, a restore and a park, for an op that runs in milliseconds. After the turn's own batch, the processor keeps taking requests queued for its model for up to OPEN_RL_TURN_DRAIN_SECONDS (default 5; 0 turns it off). It checks the non-blocking queued() before the blocking pop, so an empty queue costs nothing; a save in a batch ends the drain, since what queued behind it must not run before it; and rank 0 decides each round and broadcasts it, so every rank runs the same ops. Measured on a 20-job sweep on 8 x H100 (10 full fine-tuning jobs time-sliced 4 to 5 per GPU): lone optim_step turns went from about a third of all turns (27 and 25 in two sweeps) to none, with time between steps unchanged.
1 parent 9b32c78 commit 80d4963

4 files changed

Lines changed: 161 additions & 2 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ UNIT_TESTS ?= tests.test_session_lifecycle tests.test_proto_codec tests.test_for
2323
# Only forward BASE_URL to e2e when the user supplied it. The Makefile default
2424
# is for local CLI usage; e2e should start its own backend by default.
2525
TRAINING_TEST_BASE_URL ?= $(if $(filter environment command line,$(origin BASE_URL)),$(BASE_URL),)
26-
UNIT_TESTS += tests.test_observability_kubernetes tests.test_summaries
26+
UNIT_TESTS += tests.test_observability_kubernetes tests.test_summaries tests.test_turn_drain
2727
TRAINING_TEST_EXTRA ?= gpu
2828
TRAINING_TEST_ARGS ?=
2929
PIGLATIN_TEST_PYTHONPATH ?= examples/sft/pig-latin

‎src/server/training_requests_processor.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,12 @@ def older_versions(path: str, keep: int) -> list[str]:
8484
return versions[: max(0, len(versions) - keep)]
8585

8686

87+
# How long, after the batch a turn was taken for, a time-sliced trainer keeps taking requests that queued for its model
88+
# before it releases the GPU. Requests that arrive while the turn waits or runs (an optim_step sent right after its forward_backward) join it
89+
# instead of waiting for a turn of their own behind every other worker on the GPU. 0 turns the drain off.
90+
TURN_DRAIN_SECONDS = float(os.getenv("OPEN_RL_TURN_DRAIN_SECONDS", "5"))
91+
92+
8793
class TrainingRequestsProcessor:
8894
"""Drains training commands for one worker.
8995
@@ -120,6 +126,7 @@ def __init__(
120126
self.snapshot_registered = False
121127
self.state: StateStore | None = None # opened on first use; tests inject one
122128
self.ready_announced_at = float("-inf")
129+
self.shutdown_drained = False # a shutdown_workers taken while draining a turn; run_once honours it after answering
123130

124131
async def run(self) -> None:
125132
print(f"[WORKER] Training requests processor started (model={self.model_id} active_set={self.active_tenant_set_id}).")
@@ -199,6 +206,17 @@ async def next_batch(self) -> list[dict[str, Any]]:
199206
batch = await asyncio.to_thread(broadcast_object, batch)
200207
return batch or []
201208

209+
async def queued_now(self, deadline: float) -> list[dict[str, Any]]:
210+
"""Requests already queued for this model, without waiting for one, or none once the deadline has passed. Rank 0
211+
decides and fans the result out, as next_batch does, so every rank runs the same sequence."""
212+
batch = None
213+
if is_primary():
214+
ready = self.dedicated and time.monotonic() < deadline and await self.store.queued(self.model_id) > 0
215+
batch = await self.store.get_requests_for_model(self.model_id) if ready else []
216+
if is_distributed():
217+
batch = await asyncio.to_thread(broadcast_object, batch)
218+
return batch or []
219+
202220
async def fetch_requests(self) -> list[dict[str, Any]]:
203221
if self.dedicated:
204222
return await self.store.get_requests_for_model(self.model_id)
@@ -243,7 +261,7 @@ async def run_once(self) -> None:
243261
# from restarting on fresh weights mid-run; the run fails on its next call.
244262
print(f"[WORKER] Time slicer could not park this process: {self.time_slicer.faulted}. Exiting to free the accelerator.")
245263
await self.exit_gracefully(unregister=False)
246-
if shutdown:
264+
if shutdown or self.shutdown_drained:
247265
await self.exit_gracefully()
248266

249267
async def answer_batch(self, requests: list[dict[str, Any]]) -> tuple[list[tuple[str | None, dict[str, Any]]], Exception | None]:
@@ -278,6 +296,22 @@ async def handle_batch(self, requests: list[dict[str, Any]], results: list[tuple
278296
try:
279297
for request in gpu_reqs:
280298
results.append(await self.handle_request(request))
299+
# Requests that queued while this turn waited or ran join it. The budget counts from here, after the batch
300+
# the turn was taken for, which on a busy GPU can itself run far longer than the budget; it bounds only the
301+
# work the drain adds. A save ends the drain: as within any batch, saves run after its GPU ops, so nothing
302+
# queued after a save may run before it.
303+
deadline = time.monotonic() + TURN_DRAIN_SECONDS
304+
while not save_reqs:
305+
more = await self.queued_now(deadline)
306+
if not more:
307+
break
308+
self.shutdown_drained |= any(r.get("op") == "shutdown_workers" for r in more)
309+
more = [r for r in more if r.get("op") != "shutdown_workers"]
310+
requests.extend(more) # so a failure answers them too
311+
popped(more)
312+
save_reqs.extend(r for r in more if r.get("op") in save_ops)
313+
for request in (r for r in more if r.get("op") not in save_ops):
314+
results.append(await self.handle_request(request))
281315
finally:
282316
await asyncio.to_thread(self.worker.sleep)
283317

‎tests/test_trainer_optimizer_correctness.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,9 @@ def __init__(self, batches, events=None):
187187
self.batches = list(batches)
188188
self.queried_model_ids = []
189189

190+
async def queued(self, model_id):
191+
return 0 # nothing waits between batches, so a turn takes no more than its batch
192+
190193
async def get_requests_for_model(self, model_id):
191194
self.queried_model_ids.append(model_id)
192195
if self.batches:

‎tests/test_turn_drain.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
"""A time-sliced trainer's turn takes in the requests that queue for its model while it waits or runs, so an optim_step
2+
sent right after its forward_backward runs in the same turn instead of waiting for a turn of its own."""
3+
4+
import asyncio
5+
import contextlib
6+
import os
7+
import sys
8+
import time
9+
import unittest
10+
from unittest.mock import patch
11+
12+
from server.store import InMemoryStore
13+
from server.training_requests_processor import TrainingRequestsProcessor
14+
from tests.test_fft_batch_failure import SlicerStub
15+
from tests.test_telemetry import ScriptedTrainer, request
16+
17+
18+
class QueueStore(InMemoryStore):
19+
"""The model's queue as a list of batches: get_requests_for_model takes the next one, queued counts what waits."""
20+
21+
def __init__(self, *batches):
22+
super().__init__()
23+
self.batches = [list(b) for b in batches]
24+
self.futures = {}
25+
self.fetches = 0
26+
27+
async def get_requests_for_model(self, model_id):
28+
self.fetches += 1
29+
return self.batches.pop(0) if self.batches else []
30+
31+
async def queued(self, model_id):
32+
return len(self.batches[0]) if self.batches else 0
33+
34+
async def set_result(self, request_id, result):
35+
self.futures[request_id] = result
36+
37+
38+
class CountingSlicer(SlicerStub):
39+
def __init__(self):
40+
self.turns = 0
41+
42+
@contextlib.asynccontextmanager
43+
async def acquire(self, workload):
44+
self.turns += 1
45+
yield
46+
47+
48+
def serve(store, slicer):
49+
with patch.dict(os.environ, {"REDIS_URL": "redis://test"}):
50+
processor = TrainingRequestsProcessor(store, ScriptedTrainer(), "run-a", time_slicer=slicer)
51+
asyncio.run(processor.run_once())
52+
return processor
53+
54+
55+
class TurnDrainTest(unittest.TestCase):
56+
def test_a_request_queued_during_the_turn_runs_in_it(self) -> None:
57+
store, slicer = QueueStore([request("fb-1", "forward_backward", data=[])], [request("os-1", "optim_step")]), CountingSlicer()
58+
serve(store, slicer)
59+
self.assertEqual(slicer.turns, 1)
60+
self.assertEqual(sorted(store.futures), ["fb-1", "os-1"])
61+
self.assertNotEqual(store.futures["os-1"].get("type"), "RequestFailedResponse")
62+
63+
def test_the_drain_keeps_going_while_requests_keep_queueing(self) -> None:
64+
batches = [[request(f"fb-{i}", "forward_backward", data=[])] for i in range(3)] + [[request("os-1", "optim_step")]]
65+
store, slicer = QueueStore(*batches), CountingSlicer()
66+
serve(store, slicer)
67+
self.assertEqual((slicer.turns, sorted(store.futures)), (1, ["fb-0", "fb-1", "fb-2", "os-1"]))
68+
69+
def test_no_budget_means_no_drain(self) -> None:
70+
store, slicer = QueueStore([request("fb-1", "forward_backward", data=[])], [request("os-1", "optim_step")]), CountingSlicer()
71+
# the class's own module: another test re-imports this module, so a reference taken at import may be stale
72+
with patch.object(sys.modules[TrainingRequestsProcessor.__module__], "TURN_DRAIN_SECONDS", 0):
73+
serve(store, slicer)
74+
self.assertEqual(sorted(store.futures), ["fb-1"], "the optim_step waits for the next batch")
75+
self.assertEqual(len(store.batches), 1)
76+
77+
def test_a_save_ends_the_drain(self) -> None:
78+
# The save in the first batch runs after the turn's GPU ops; nothing queued behind it may run before it.
79+
first = [request("os-1", "optim_step"), request("sv-1", "save_weights_for_sampler", path="x")]
80+
store, slicer = QueueStore(first, [request("fb-2", "forward_backward", data=[])]), CountingSlicer()
81+
with patch.object(TrainingRequestsProcessor, "handle_request", new=_answer):
82+
serve(store, slicer)
83+
self.assertEqual(store.fetches, 1)
84+
self.assertEqual(len(store.batches), 1, "fb-2 stays queued for the next batch")
85+
86+
def test_an_empty_queue_is_not_waited_on(self) -> None:
87+
store, slicer = QueueStore([request("fb-1", "forward_backward", data=[])]), CountingSlicer()
88+
serve(store, slicer)
89+
self.assertEqual(store.fetches, 1, "an empty queue is checked with queued(), never with the blocking pop")
90+
91+
def test_a_shutdown_taken_while_draining_is_honoured(self) -> None:
92+
store = QueueStore([request("fb-1", "forward_backward", data=[])], [{"request_id": "sd", "op": "shutdown_workers", "model_id": "run-a"}])
93+
with patch.object(TrainingRequestsProcessor, "exit_gracefully") as exit_gracefully:
94+
serve(store, CountingSlicer())
95+
exit_gracefully.assert_called_once()
96+
97+
98+
async def _answer(self, r):
99+
"""Answers any request, so a save needs no real checkpoint."""
100+
return r["request_id"], {"type": "ok"}
101+
102+
103+
if __name__ == "__main__":
104+
unittest.main()
105+
106+
107+
class LongBatchTest(unittest.TestCase):
108+
def test_the_budget_starts_after_the_turns_own_batch(self) -> None:
109+
# A forward_backward that runs longer than the whole budget must still let its queued optim_step join the turn.
110+
class SlowTrainer(ScriptedTrainer):
111+
def forward_backward(self, *args, **kwargs):
112+
time.sleep(0.2)
113+
return super().forward_backward(*args, **kwargs)
114+
115+
store, slicer = QueueStore([request("fb-1", "forward_backward", data=[])], [request("os-1", "optim_step")]), CountingSlicer()
116+
with (
117+
patch.object(sys.modules[TrainingRequestsProcessor.__module__], "TURN_DRAIN_SECONDS", 0.05),
118+
patch.dict(os.environ, {"REDIS_URL": "redis://test"}),
119+
):
120+
processor = TrainingRequestsProcessor(store, SlowTrainer(), "run-a", time_slicer=slicer)
121+
asyncio.run(processor.run_once())
122+
self.assertEqual((slicer.turns, sorted(store.futures)), (1, ["fb-1", "os-1"]))

0 commit comments

Comments
 (0)