From a010ce8cff04e2a0b36a9652f07a756239680136 Mon Sep 17 00:00:00 2001 From: chenjun Date: Fri, 10 Jul 2026 10:47:38 +0000 Subject: [PATCH] CI: gate MTP acceptance rate alongside gsm8k accuracy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native ATOM accuracy pipeline already scrapes the MTP/spec-decode acceptance rate from the server log during the gsm8k run (atom_ci_metadata.mtp_acceptance_rate) and publishes it to the dashboard, but nothing gates on it. gsm8k accuracy alone cannot guard MTP: speculative decoding is lossless w.r.t. the target model, so a broken draft head leaves accuracy unchanged and only craters acceptance. Add a "Check MTP acceptance rate" step to atom-test.yaml that reads the already-recorded acceptance rate from the accuracy result JSON and fails the job when it drops below a per-model threshold (or is missing, which signals MTP stats were never emitted). No extra inference run — it reuses the same gsm8k pass as the accuracy check. Thresholds (mtp_accept_threshold, fraction 0-1) are added to all 10 ATOM-backend spec-decode entries in models_accuracy.json, set to each model's dashboard-observed minimum minus 1.5 points. FP4 MTP uses its healthy pre-regression minimum, so the current ~36% drop is caught rather than baked in. The catalog schema gains mtp_accept_threshold / mtp_per_pos_threshold so validate_catalog.py accepts the new fields. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/benchmark/models_accuracy.json | 10 +++++ .../schema/accuracy_catalog.schema.json | 3 ++ .github/workflows/atom-test.yaml | 37 +++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/.github/benchmark/models_accuracy.json b/.github/benchmark/models_accuracy.json index 6439ba4426..726704d141 100644 --- a/.github/benchmark/models_accuracy.json +++ b/.github/benchmark/models_accuracy.json @@ -43,6 +43,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "pr", "accuracy_threshold": 0.94, + "mtp_accept_threshold": 0.622, "accuracy_baseline": 0.96, "accuracy_baseline_model": "deepseek-ai/DeepSeek-V4-Pro", "_baseline_note": "Same base model as DeepSeek-V4-Pro FP8 (MTP-3)." @@ -55,6 +56,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "main", "accuracy_threshold": 0.94, + "mtp_accept_threshold": 0.62, "accuracy_baseline": 0.9553, "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528", "_baseline_note": "Same base model as DeepSeek-R1-0528 FP8" @@ -67,6 +69,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.93, + "mtp_accept_threshold": 0.582, "accuracy_baseline": 0.9553, "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528", "_baseline_note": "Online quantization on top of DeepSeek-R1-0528 MTP (FP8 native): global ptpc_fp8 + expert layers mxfp4, excluding lm_head and *.gate.*. Threshold set to 0.93 (same headroom as DeepSeek-R1-0528-FP4 MTP) as a conservative placeholder for the MoE-MXFP4 accuracy drop." @@ -116,6 +119,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "pr", "accuracy_threshold": 0.93, + "mtp_accept_threshold": 0.593, "accuracy_baseline": 0.9553, "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528", "_baseline_note": "CI measured FP8 baseline (deepseek-ai/DeepSeek-R1-0528 is natively FP8)" @@ -189,6 +193,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.91, + "mtp_accept_threshold": 0.67, "accuracy_baseline": 0.9257, "accuracy_baseline_model": "amd/Kimi-K2.5-MXFP4 + lightseekorg/kimi-k2.5-eagle3", "_baseline_note": "Eagle3 spec decode on Kimi-K2.5-MXFP4." @@ -237,6 +242,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.92, + "mtp_accept_threshold": 0.595, "accuracy_baseline": 0.9447, "accuracy_baseline_model": "zai-org/GLM-5.2-FP8", "_baseline_note": "Initial GLM-5.2-MXFP4 MTP online-quant native accuracy case. Threshold/baseline follow GLM-5.2-FP8 until MXFP4 MTP CI baseline is calibrated." @@ -285,6 +291,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "main", "accuracy_threshold": 0.85, + "mtp_accept_threshold": 0.829, "accuracy_baseline": 0.9538, "accuracy_baseline_model": "Qwen/Qwen3.5-397B-A17B-FP8", "_baseline_note": "Same base model as Qwen3.5-397B-A17B-FP8; MTP3" @@ -309,6 +316,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "main", "accuracy_threshold": 0.835, + "mtp_accept_threshold": 0.828, "accuracy_baseline": 0.9538, "accuracy_baseline_model": "Qwen/Qwen3.5-397B-A17B-FP8", "_baseline_note": "CI baseline=0.8605. HF card reports 0.9538 but uses chat API with reasoning_parser" @@ -359,6 +367,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.93, + "mtp_accept_threshold": 0.715, "accuracy_baseline": 0.9469, "accuracy_baseline_model": "amd/MiniMax-M3-MXFP4 + Inferact/MiniMax-M3-EAGLE3", "_baseline_note": "FP4 M3 + EAGLE3 draft (tp8), lossless vs greedy target." @@ -383,6 +392,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.778, + "mtp_accept_threshold": 0.82, "accuracy_baseline": 0.79, "accuracy_baseline_model": "XiaomiMiMo/MiMo-V2-Flash", "_baseline_note": "CI GSM8K 3-shot MTP1=0.7983 (run 26410931088). Baseline 0.79; threshold 0.778 (~1.1σ). tp MUST=4, num-speculative-tokens MUST=1: ATOM builds only MTP layer 0 (vLLM _MIMO_V2_FLASH_NUM_MTP_LAYERS=1); more spec → layers 1/2 KV unpopulated, accept craters." diff --git a/.github/benchmark/schema/accuracy_catalog.schema.json b/.github/benchmark/schema/accuracy_catalog.schema.json index ce45f8a7d3..3528571de7 100644 --- a/.github/benchmark/schema/accuracy_catalog.schema.json +++ b/.github/benchmark/schema/accuracy_catalog.schema.json @@ -29,6 +29,9 @@ "accuracy_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, "accuracy_test_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "mtp_accept_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "mtp_per_pos_threshold": { "type": "string" }, + "accuracy_baseline_model": { "type": "string" }, "client_command": { "type": "string" }, "priority": { "type": "string", "enum": ["P0", "P1", "P2"] }, diff --git a/.github/workflows/atom-test.yaml b/.github/workflows/atom-test.yaml index 01ad738663..8b9c4d3bbe 100644 --- a/.github/workflows/atom-test.yaml +++ b/.github/workflows/atom-test.yaml @@ -450,6 +450,43 @@ jobs: echo "Accuracy test passed: $flexible_extract_value >= $threshold" fi + - name: Check MTP acceptance rate + # Only gates models that declare mtp_accept_threshold in the catalog + # (MTP/spec-decode entries); all other models skip this step. + if: ${{ success() && matrix.mtp_accept_threshold != '' }} + env: + MTP_ACCEPT_THRESHOLD: ${{ matrix.mtp_accept_threshold }} + run: | + # gsm8k accuracy alone CANNOT guard MTP: speculative decoding is + # lossless w.r.t. the target model, so a broken draft head leaves + # accuracy unchanged and only craters acceptance/throughput. This + # step is the only gate that catches such an MTP regression. + # atom_test.sh scrapes the cumulative "[MTP Stats ]" line from the + # server log during the gsm8k run and records mtp_acceptance_rate + # (a PERCENT, e.g. 62.5) into atom_ci_metadata. The catalog + # threshold is a FRACTION (0-1, same convention as accuracy_threshold), + # so we divide the recorded percent by 100 before comparing. + result_file=$(ls -1t accuracy_test_results/*.json 2>/dev/null | head -n 1) + if [ -z "$result_file" ] || [ ! -f "$result_file" ]; then + echo "ERROR: No results JSON file found for MTP acceptance check." + exit 2 + fi + echo "RESULT_FILE: $result_file" + + rate=$(jq -r '.atom_ci_metadata.mtp_acceptance_rate // empty' "$result_file") + if [ -z "$rate" ]; then + echo "FAIL: mtp_acceptance_rate missing — MTP stats were not emitted" + echo " during the accuracy run. Treating as an MTP regression." + exit 1 + fi + + echo "MTP acceptance: ${rate}% (threshold ${MTP_ACCEPT_THRESHOLD} = $(awk -v t="$MTP_ACCEPT_THRESHOLD" 'BEGIN{printf "%.1f", t*100}')%)" + if awk -v v="$rate" -v t="$MTP_ACCEPT_THRESHOLD" 'BEGIN {exit !((v/100) < t)}'; then + echo "MTP acceptance gate FAILED: ${rate}% < threshold" + exit 1 + fi + echo "MTP acceptance gate PASSED." + - name: Collect Test Summary if: success() env: