CI: gate MTP acceptance rate alongside gsm8k accuracy

.github/benchmark/models_accuracy.json

@@ -43,6 +43,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "pr",
     "accuracy_threshold": 0.94,
+    "mtp_accept_threshold": 0.622,
     "accuracy_baseline": 0.96,
     "accuracy_baseline_model": "deepseek-ai/DeepSeek-V4-Pro",
     "_baseline_note": "Same base model as DeepSeek-V4-Pro FP8 (MTP-3)."
@@ -55,6 +56,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "main",
     "accuracy_threshold": 0.94,
+    "mtp_accept_threshold": 0.62,
     "accuracy_baseline": 0.9553,
     "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528",
     "_baseline_note": "Same base model as DeepSeek-R1-0528 FP8"
@@ -67,6 +69,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "nightly",
     "accuracy_threshold": 0.93,
+    "mtp_accept_threshold": 0.582,
     "accuracy_baseline": 0.9553,
     "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528",
     "_baseline_note": "Online quantization on top of DeepSeek-R1-0528 MTP (FP8 native): global ptpc_fp8 + expert layers mxfp4, excluding lm_head and *.gate.*. Threshold set to 0.93 (same headroom as DeepSeek-R1-0528-FP4 MTP) as a conservative placeholder for the MoE-MXFP4 accuracy drop."
@@ -116,6 +119,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "pr",
     "accuracy_threshold": 0.93,
+    "mtp_accept_threshold": 0.593,
     "accuracy_baseline": 0.9553,
     "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528",
     "_baseline_note": "CI measured FP8 baseline (deepseek-ai/DeepSeek-R1-0528 is natively FP8)"
@@ -189,6 +193,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "nightly",
     "accuracy_threshold": 0.91,
+    "mtp_accept_threshold": 0.67,
     "accuracy_baseline": 0.9257,
     "accuracy_baseline_model": "amd/Kimi-K2.5-MXFP4 + lightseekorg/kimi-k2.5-eagle3",
     "_baseline_note": "Eagle3 spec decode on Kimi-K2.5-MXFP4."
@@ -237,6 +242,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "nightly",
     "accuracy_threshold": 0.92,
+    "mtp_accept_threshold": 0.595,
     "accuracy_baseline": 0.9447,
     "accuracy_baseline_model": "zai-org/GLM-5.2-FP8",
     "_baseline_note": "Initial GLM-5.2-MXFP4 MTP online-quant native accuracy case. Threshold/baseline follow GLM-5.2-FP8 until MXFP4 MTP CI baseline is calibrated."
@@ -285,6 +291,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "main",
     "accuracy_threshold": 0.85,
+    "mtp_accept_threshold": 0.829,
     "accuracy_baseline": 0.9538,
     "accuracy_baseline_model": "Qwen/Qwen3.5-397B-A17B-FP8",
     "_baseline_note": "Same base model as Qwen3.5-397B-A17B-FP8; MTP3"
@@ -309,6 +316,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "main",
     "accuracy_threshold": 0.835,
+    "mtp_accept_threshold": 0.828,
     "accuracy_baseline": 0.9538,
     "accuracy_baseline_model": "Qwen/Qwen3.5-397B-A17B-FP8",
     "_baseline_note": "CI baseline=0.8605. HF card reports 0.9538 but uses chat API with reasoning_parser"
@@ -359,6 +367,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "nightly",
     "accuracy_threshold": 0.93,
+    "mtp_accept_threshold": 0.715,
     "accuracy_baseline": 0.9469,
     "accuracy_baseline_model": "amd/MiniMax-M3-MXFP4 + Inferact/MiniMax-M3-EAGLE3",
     "_baseline_note": "FP4 M3 + EAGLE3 draft (tp8), lossless vs greedy target."
@@ -383,6 +392,7 @@
     "runner": "linux-atom-do-mi350x-8",
     "test_level": "nightly",
     "accuracy_threshold": 0.778,
+    "mtp_accept_threshold": 0.82,
     "accuracy_baseline": 0.79,
     "accuracy_baseline_model": "XiaomiMiMo/MiMo-V2-Flash",
     "_baseline_note": "CI GSM8K 3-shot MTP1=0.7983 (run 26410931088). Baseline 0.79; threshold 0.778 (~1.1σ). tp MUST=4, num-speculative-tokens MUST=1: ATOM builds only MTP layer 0 (vLLM _MIMO_V2_FLASH_NUM_MTP_LAYERS=1); more spec → layers 1/2 KV unpopulated, accept craters."

.github/benchmark/schema/accuracy_catalog.schema.json

@@ -29,6 +29,9 @@
       "accuracy_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
       "accuracy_test_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
 
+      "mtp_accept_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
+      "mtp_per_pos_threshold": { "type": "string" },
+
       "accuracy_baseline_model": { "type": "string" },
       "client_command": { "type": "string" },
       "priority": { "type": "string", "enum": ["P0", "P1", "P2"] },

.github/workflows/atom-test.yaml

@@ -450,6 +450,43 @@ jobs:
             echo "Accuracy test passed: $flexible_extract_value >= $threshold"
           fi
 
+      - name: Check MTP acceptance rate
+        # Only gates models that declare mtp_accept_threshold in the catalog
+        # (MTP/spec-decode entries); all other models skip this step.
+        if: ${{ success() && matrix.mtp_accept_threshold != '' }}
+        env:
+          MTP_ACCEPT_THRESHOLD: ${{ matrix.mtp_accept_threshold }}
+        run: |
+          # gsm8k accuracy alone CANNOT guard MTP: speculative decoding is
+          # lossless w.r.t. the target model, so a broken draft head leaves
+          # accuracy unchanged and only craters acceptance/throughput. This
+          # step is the only gate that catches such an MTP regression.
+          # atom_test.sh scrapes the cumulative "[MTP Stats ]" line from the
+          # server log during the gsm8k run and records mtp_acceptance_rate
+          # (a PERCENT, e.g. 62.5) into atom_ci_metadata. The catalog
+          # threshold is a FRACTION (0-1, same convention as accuracy_threshold),
+          # so we divide the recorded percent by 100 before comparing.
+          result_file=$(ls -1t accuracy_test_results/*.json 2>/dev/null | head -n 1)
+          if [ -z "$result_file" ] || [ ! -f "$result_file" ]; then
+            echo "ERROR: No results JSON file found for MTP acceptance check."
+            exit 2
+          fi
+          echo "RESULT_FILE: $result_file"
+
+          rate=$(jq -r '.atom_ci_metadata.mtp_acceptance_rate // empty' "$result_file")
+          if [ -z "$rate" ]; then
+            echo "FAIL: mtp_acceptance_rate missing — MTP stats were not emitted"
+            echo "      during the accuracy run. Treating as an MTP regression."
+            exit 1
+          fi
+
+          echo "MTP acceptance: ${rate}% (threshold ${MTP_ACCEPT_THRESHOLD} = $(awk -v t="$MTP_ACCEPT_THRESHOLD" 'BEGIN{printf "%.1f", t*100}')%)"
+          if awk -v v="$rate" -v t="$MTP_ACCEPT_THRESHOLD" 'BEGIN {exit !((v/100) < t)}'; then
+            echo "MTP acceptance gate FAILED: ${rate}% < threshold"
+            exit 1
+          fi
+          echo "MTP acceptance gate PASSED."
+
       - name: Collect Test Summary
         if: success()
         env: