diff --git a/.github/benchmark/models_accuracy.json b/.github/benchmark/models_accuracy.json index 6439ba4426..726704d141 100644 --- a/.github/benchmark/models_accuracy.json +++ b/.github/benchmark/models_accuracy.json @@ -43,6 +43,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "pr", "accuracy_threshold": 0.94, + "mtp_accept_threshold": 0.622, "accuracy_baseline": 0.96, "accuracy_baseline_model": "deepseek-ai/DeepSeek-V4-Pro", "_baseline_note": "Same base model as DeepSeek-V4-Pro FP8 (MTP-3)." @@ -55,6 +56,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "main", "accuracy_threshold": 0.94, + "mtp_accept_threshold": 0.62, "accuracy_baseline": 0.9553, "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528", "_baseline_note": "Same base model as DeepSeek-R1-0528 FP8" @@ -67,6 +69,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.93, + "mtp_accept_threshold": 0.582, "accuracy_baseline": 0.9553, "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528", "_baseline_note": "Online quantization on top of DeepSeek-R1-0528 MTP (FP8 native): global ptpc_fp8 + expert layers mxfp4, excluding lm_head and *.gate.*. Threshold set to 0.93 (same headroom as DeepSeek-R1-0528-FP4 MTP) as a conservative placeholder for the MoE-MXFP4 accuracy drop." @@ -116,6 +119,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "pr", "accuracy_threshold": 0.93, + "mtp_accept_threshold": 0.593, "accuracy_baseline": 0.9553, "accuracy_baseline_model": "deepseek-ai/DeepSeek-R1-0528", "_baseline_note": "CI measured FP8 baseline (deepseek-ai/DeepSeek-R1-0528 is natively FP8)" @@ -189,6 +193,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.91, + "mtp_accept_threshold": 0.67, "accuracy_baseline": 0.9257, "accuracy_baseline_model": "amd/Kimi-K2.5-MXFP4 + lightseekorg/kimi-k2.5-eagle3", "_baseline_note": "Eagle3 spec decode on Kimi-K2.5-MXFP4." @@ -237,6 +242,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.92, + "mtp_accept_threshold": 0.595, "accuracy_baseline": 0.9447, "accuracy_baseline_model": "zai-org/GLM-5.2-FP8", "_baseline_note": "Initial GLM-5.2-MXFP4 MTP online-quant native accuracy case. Threshold/baseline follow GLM-5.2-FP8 until MXFP4 MTP CI baseline is calibrated." @@ -285,6 +291,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "main", "accuracy_threshold": 0.85, + "mtp_accept_threshold": 0.829, "accuracy_baseline": 0.9538, "accuracy_baseline_model": "Qwen/Qwen3.5-397B-A17B-FP8", "_baseline_note": "Same base model as Qwen3.5-397B-A17B-FP8; MTP3" @@ -309,6 +316,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "main", "accuracy_threshold": 0.835, + "mtp_accept_threshold": 0.828, "accuracy_baseline": 0.9538, "accuracy_baseline_model": "Qwen/Qwen3.5-397B-A17B-FP8", "_baseline_note": "CI baseline=0.8605. HF card reports 0.9538 but uses chat API with reasoning_parser" @@ -359,6 +367,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.93, + "mtp_accept_threshold": 0.715, "accuracy_baseline": 0.9469, "accuracy_baseline_model": "amd/MiniMax-M3-MXFP4 + Inferact/MiniMax-M3-EAGLE3", "_baseline_note": "FP4 M3 + EAGLE3 draft (tp8), lossless vs greedy target." @@ -383,6 +392,7 @@ "runner": "linux-atom-do-mi350x-8", "test_level": "nightly", "accuracy_threshold": 0.778, + "mtp_accept_threshold": 0.82, "accuracy_baseline": 0.79, "accuracy_baseline_model": "XiaomiMiMo/MiMo-V2-Flash", "_baseline_note": "CI GSM8K 3-shot MTP1=0.7983 (run 26410931088). Baseline 0.79; threshold 0.778 (~1.1σ). tp MUST=4, num-speculative-tokens MUST=1: ATOM builds only MTP layer 0 (vLLM _MIMO_V2_FLASH_NUM_MTP_LAYERS=1); more spec → layers 1/2 KV unpopulated, accept craters." diff --git a/.github/benchmark/schema/accuracy_catalog.schema.json b/.github/benchmark/schema/accuracy_catalog.schema.json index ce45f8a7d3..3528571de7 100644 --- a/.github/benchmark/schema/accuracy_catalog.schema.json +++ b/.github/benchmark/schema/accuracy_catalog.schema.json @@ -29,6 +29,9 @@ "accuracy_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, "accuracy_test_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "mtp_accept_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "mtp_per_pos_threshold": { "type": "string" }, + "accuracy_baseline_model": { "type": "string" }, "client_command": { "type": "string" }, "priority": { "type": "string", "enum": ["P0", "P1", "P2"] }, diff --git a/.github/workflows/atom-test.yaml b/.github/workflows/atom-test.yaml index 01ad738663..8b9c4d3bbe 100644 --- a/.github/workflows/atom-test.yaml +++ b/.github/workflows/atom-test.yaml @@ -450,6 +450,43 @@ jobs: echo "Accuracy test passed: $flexible_extract_value >= $threshold" fi + - name: Check MTP acceptance rate + # Only gates models that declare mtp_accept_threshold in the catalog + # (MTP/spec-decode entries); all other models skip this step. + if: ${{ success() && matrix.mtp_accept_threshold != '' }} + env: + MTP_ACCEPT_THRESHOLD: ${{ matrix.mtp_accept_threshold }} + run: | + # gsm8k accuracy alone CANNOT guard MTP: speculative decoding is + # lossless w.r.t. the target model, so a broken draft head leaves + # accuracy unchanged and only craters acceptance/throughput. This + # step is the only gate that catches such an MTP regression. + # atom_test.sh scrapes the cumulative "[MTP Stats ]" line from the + # server log during the gsm8k run and records mtp_acceptance_rate + # (a PERCENT, e.g. 62.5) into atom_ci_metadata. The catalog + # threshold is a FRACTION (0-1, same convention as accuracy_threshold), + # so we divide the recorded percent by 100 before comparing. + result_file=$(ls -1t accuracy_test_results/*.json 2>/dev/null | head -n 1) + if [ -z "$result_file" ] || [ ! -f "$result_file" ]; then + echo "ERROR: No results JSON file found for MTP acceptance check." + exit 2 + fi + echo "RESULT_FILE: $result_file" + + rate=$(jq -r '.atom_ci_metadata.mtp_acceptance_rate // empty' "$result_file") + if [ -z "$rate" ]; then + echo "FAIL: mtp_acceptance_rate missing — MTP stats were not emitted" + echo " during the accuracy run. Treating as an MTP regression." + exit 1 + fi + + echo "MTP acceptance: ${rate}% (threshold ${MTP_ACCEPT_THRESHOLD} = $(awk -v t="$MTP_ACCEPT_THRESHOLD" 'BEGIN{printf "%.1f", t*100}')%)" + if awk -v v="$rate" -v t="$MTP_ACCEPT_THRESHOLD" 'BEGIN {exit !((v/100) < t)}'; then + echo "MTP acceptance gate FAILED: ${rate}% < threshold" + exit 1 + fi + echo "MTP acceptance gate PASSED." + - name: Collect Test Summary if: success() env: