diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py
index 12867437d9dd..94496984c580 100755
--- a/deepspeed/runtime/engine.py
+++ b/deepspeed/runtime/engine.py
@@ -3601,6 +3601,7 @@ def _save_moe_checkpoint(self, save_dir, tag, client_state={}, exclude_frozen_pa
                     local_expert_id = None
                     if not m:
                         logger.warning(f'No expert found in key {key}.')
+                        continue
                     else:
                         local_expert_id = m.group(1)