Merge main into PR #2351

Signed-off-by: 0z5a <dezhen.lu@student.uni-tuebingen.de>

Author: 0z5a

.github/workflows/pr-test.yml

@@ -265,8 +265,8 @@ jobs:
               "test_file": "test_qwen3.5_0.8B_gsm8k_short.py"
             },
             {
-              "num_gpus": 4,
-              "test_file": "test_qwen3.5_0.8B_gsm8k_async_short.py"
+              "num_gpus": 2,
+              "test_file": "test_qwen2.5_0.5B_score_centering.py"
             },
             {
               "num_gpus": 8,
@@ -673,6 +673,18 @@ jobs:
               "num_gpus": 0,
               "test_file": "test_policy_loss.py"
             },
+            {
+              "num_gpus": 0,
+              "test_file": "test_score_centering.py"
+            },
+            {
+              "num_gpus": 0,
+              "test_file": "test_score_centering_binary.py"
+            },
+            {
+              "num_gpus": 0,
+              "test_file": "test_score_centering_transport.py"
+            },
             {
               "num_gpus": 0,
               "test_file": "test_ppo_logprob_entropy.py"
@@ -733,6 +745,22 @@ jobs:
               "num_gpus": 0,
               "test_file": "test_fully_async_rollout.py"
             },
+            {
+              "num_gpus": 0,
+              "test_file": "test_rollout_buffer_order.py"
+            },
+            {
+              "num_gpus": 0,
+              "test_file": "test_server_control.py"
+            },
+            {
+              "num_gpus": 0,
+              "test_file": "test_cuda_stack_offload.py"
+            },
+            {
+              "num_gpus": 0,
+              "test_file": "test_r3_spill.py"
+            },
             {
               "num_gpus": 0,
               "test_file": "test_rollout_sample_hooks.py"
@@ -1097,8 +1125,8 @@ jobs:
               "test_file": "test_qwen3.5_0.8B_gsm8k_short.py"
             },
             {
-              "num_gpus": 4,
-              "test_file": "test_qwen3.5_0.8B_gsm8k_async_short.py"
+              "num_gpus": 2,
+              "test_file": "test_qwen2.5_0.5B_score_centering.py"
             },
             {
               "num_gpus": 8,
@@ -1227,7 +1255,7 @@ jobs:
             FIRST=true
             for filepath in $CHANGED; do
               # Extract NUM_GPUS from the test file, default to 8
-              NGPU=$(grep -oP '^NUM_GPUS\s*=\s*\K\d+' "$filepath" | head -1)
+              NGPU=$(grep -m1 -oP '^NUM_GPUS\s*=\s*\K\d+' "$filepath" || true)
               NGPU=${NGPU:-8}
               if [ "$FIRST" = true ]; then FIRST=false; else MATRIX+=","; fi
               MATRIX+="{\"test_file\":\"$filepath\",\"num_gpus\":$NGPU}"

.github/workflows/pr-test.yml.j2

@@ -24,7 +24,7 @@
     {'test_file': 'test_qwen2.5_0.5B_fanout_short.py', 'num_gpus': 4},
     {'test_file': 'test_qwen3_4B_streaming_partial_rollout.py', 'num_gpus': 8},
     {'test_file': 'test_qwen3.5_0.8B_gsm8k_short.py', 'num_gpus': 4},
-    {'test_file': 'test_qwen3.5_0.8B_gsm8k_async_short.py', 'num_gpus': 4},
+    {'test_file': 'test_qwen2.5_0.5B_score_centering.py', 'num_gpus': 2},
     {'test_file': 'test_qwen3_4B_ckpt.py', 'test_args': '--save-optimizer gpu --load-optimizer gpu', 'num_gpus': 8},
     {'test_file': 'test_qwen3_4B_ckpt.py', 'test_args': '--save-optimizer gpu --load-optimizer cpu', 'num_gpus': 8},
     {'test_file': 'test_qwen3_4B_ckpt.py', 'test_args': '--save-optimizer cpu --load-optimizer cpu', 'num_gpus': 8},
@@ -94,6 +94,9 @@
         {'test_file': 'test_ppo_kl_metric.py', 'num_gpus': 0},
         {'test_file': 'test_cispo_loss.py', 'num_gpus': 0},
         {'test_file': 'test_policy_loss.py', 'num_gpus': 0},
+        {'test_file': 'test_score_centering.py', 'num_gpus': 0},
+        {'test_file': 'test_score_centering_binary.py', 'num_gpus': 0},
+        {'test_file': 'test_score_centering_transport.py', 'num_gpus': 0},
         {'test_file': 'test_ppo_logprob_entropy.py', 'num_gpus': 0},
         {'test_file': 'test_discounted_returns.py', 'num_gpus': 0},
         {'test_file': 'test_block_fp8_zero_block.py', 'num_gpus': 0},
@@ -109,6 +112,10 @@
         {'test_file': 'test_filter_long_prompt.py', 'num_gpus': 0},
         {'test_file': 'test_read_file_slicing.py', 'num_gpus': 0},
         {'test_file': 'test_fully_async_rollout.py', 'num_gpus': 0},
+        {'test_file': 'test_rollout_buffer_order.py', 'num_gpus': 0},
+        {'test_file': 'test_server_control.py', 'num_gpus': 0},
+        {'test_file': 'test_cuda_stack_offload.py', 'num_gpus': 0},
+        {'test_file': 'test_r3_spill.py', 'num_gpus': 0},
         {'test_file': 'test_rollout_sample_hooks.py', 'num_gpus': 0},
         {'test_file': 'test_hf_to_megatron.py', 'num_gpus': 0},
         {'test_file': 'test_qwen3_5_vl_native.py', 'num_gpus': 0},
@@ -334,7 +341,7 @@ jobs:
             FIRST=true
             for filepath in $CHANGED; do
               # Extract NUM_GPUS from the test file, default to 8
-              NGPU=$(grep -oP '^NUM_GPUS\s*=\s*\K\d+' "$filepath" | head -1)
+              NGPU=$(grep -m1 -oP '^NUM_GPUS\s*=\s*\K\d+' "$filepath" || true)
               NGPU=${NGPU:-8}
               if [ "$FIRST" = true ]; then FIRST=false; else MATRIX+=","; fi
               MATRIX+="{\"test_file\":\"$filepath\",\"num_gpus\":$NGPU}"

README.md

@@ -51,7 +51,7 @@ Choosing SGLang as the single rollout backend is also intentional. Multi-backend
 
 ## Correctness, Stability, and CI
 
-slime is developed as RL infrastructure, where "the script runs" is not enough. The project maintains CPU unit tests, contract tests for customization hooks, and GPU end-to-end tests covering dense and MoE models, Megatron training paths, SGLang deployment configurations, checkpointing, numerical precision, async rollout, OPD, PPO-style workflows, and debug rollout-then-train replay.
+slime is developed as RL infrastructure, where "the script runs" is not enough. The project maintains CPU unit tests, contract tests for customization hooks, and GPU end-to-end tests covering dense and MoE models, Megatron training paths, SGLang deployment configurations, checkpointing, numerical precision, fully-async rollout, OPD, PPO-style workflows, and debug rollout-then-train replay.
 
 Useful engineering docs:
 
@@ -131,7 +131,7 @@ These are not just demos. They are independent systems that use slime as a reusa
 
 ### 🦞 OpenClaw-RL: Train a Personalized Clawbot Simply by Talking to It
 
-[**OpenClaw-RL**](https://github.com/Gen-Verse/OpenClaw-RL) is an RL server for personalized OpenClaw agents. It hosts the OpenClaw model and improves it from prior conversations across deployments, while slime's asynchronous RL infrastructure prevents training from interfering with API serving. It supports two automatic optimization methods: GRPO with binary feedback inferred from subsequent states, and on-policy distillation that extracts hindsight hints from later feedback for the current policy.
+[**OpenClaw-RL**](https://github.com/Gen-Verse/OpenClaw-RL) is an RL server for personalized OpenClaw agents. It hosts the OpenClaw model and improves it from prior conversations across deployments, while slime's decoupled RL infrastructure prevents training from interfering with API serving. It supports two automatic optimization methods: GRPO with binary feedback inferred from subsequent states, and on-policy distillation that extracts hindsight hints from later feedback for the current policy.
 
 ### ⚛️ P1: Mastering Physics Olympiads with Reinforcement Learning
 

README_zh.md

@@ -51,7 +51,7 @@ slime 不只是一个能调用推理后端的框架。它尽量保留 Megatron 
 
 ## 正确性、稳定性与 CI
 
-slime 被当作 RL 基础设施来开发，因为“脚本能跑起来”远远不够。项目维护 CPU unit test、customization hook contract test，以及 GPU end-to-end test，覆盖 dense 和 MoE 模型、Megatron training path、SGLang deployment config、checkpoint、数值精度、async rollout、OPD、PPO-style workflow，以及 debug rollout-then-train replay。
+slime 被当作 RL 基础设施来开发，因为“脚本能跑起来”远远不够。项目维护 CPU unit test、customization hook contract test，以及 GPU end-to-end test，覆盖 dense 和 MoE 模型、Megatron training path、SGLang deployment config、checkpoint、数值精度、fully-async rollout、OPD、PPO-style workflow，以及 debug rollout-then-train replay。
 
 相关工程文档：
 
@@ -133,7 +133,7 @@ slime 被当作 RL 基础设施来开发，因为“脚本能跑起来”远远
 
 ### 🦞 OpenClaw-RL: Train a Personalized Clawbot Simply by Talking to It
 
-[**OpenClaw-RL**](https://github.com/Gen-Verse/OpenClaw-RL) 是面向 personalized OpenClaw agent 的 RL server。它托管 OpenClaw model，并从跨部署的历史对话中持续改进模型，同时 slime 的 asynchronous RL infrastructure 避免训练过程干扰 API serving。它支持两种自动优化方法：基于后续状态推断 binary feedback 的 GRPO，以及从后续反馈中提取 hindsight hint 的 on-policy distillation。
+[**OpenClaw-RL**](https://github.com/Gen-Verse/OpenClaw-RL) 是面向 personalized OpenClaw agent 的 RL server。它托管 OpenClaw model，并从跨部署的历史对话中持续改进模型，同时 slime 的 decoupled RL infrastructure 避免训练过程干扰 API serving。它支持两种自动优化方法：基于后续状态推断 binary feedback 的 GRPO，以及从后续反馈中提取 hindsight hint 的 on-policy distillation。
 
 ### ⚛️ P1: Mastering Physics Olympiads with Reinforcement Learning
 

docker/npu_patch/slime.patch

@@ -678,14 +678,13 @@ diff --git a/slime/utils/external_utils/command_utils.py b/slime/utils/external_
 index 9f51ecdf..d4b47eca 100644
 --- a/slime/utils/external_utils/command_utils.py
 +++ b/slime/utils/external_utils/command_utils.py
-@@ -184,6 +184,108 @@ def execute_train(
+@@ -184,6 +184,107 @@ def execute_train(
              f"{train_args}"
          )
 
 +def execute_train_npu(
 +    train_args: str,
 +    megatron_model_type: str | None,
-+    train_script: str = "train.py",
 +    before_ray_job_submit=None,
 +    extra_env_vars=None,
 +    config: ExecuteTrainConfig | None = None,
@@ -780,7 +779,7 @@ index 9f51ecdf..d4b47eca 100644
 +            f"{cmd_megatron_model_source}"
 +            f'ray job submit --address="http://127.0.0.1:8265" '
 +            f"--runtime-env-json='{runtime_env_json}' "
-+            f"-- python3 {train_script} "
++            f"-- python3 train.py "
 +            f"{'${MODEL_ARGS[@]}' if megatron_model_type is not None else ''} "
 +            f"{train_args}"
 +        )