diff --git a/multilingual_t5/tasks.py b/multilingual_t5/tasks.py
index b3abc1a..e2a489e 100644
--- a/multilingual_t5/tasks.py
+++ b/multilingual_t5/tasks.py
@@ -358,10 +358,12 @@ def create_xnli_tasks_and_mixtures(task_prefix, task_suffix, output_features):
       output_features=DEFAULT_OUTPUT_FEATURES,
       metric_fns=[metrics.squad])
 
-tydiqa = (["mt5_tydiqa_train_dev"] +
-          ["mt5_tydiqa_dev.{}".format(lang) for lang in TYDIQA_LANGS])
+tydiqa_dev = ["mt5_tydiqa_dev.{}".format(lang) for lang in TYDIQA_LANGS]
+tydiqa = ["mt5_tydiqa_train_dev"] + tydiqa_dev
 seqio.MixtureRegistry.add("mt5_tydiqa", tydiqa, default_rate=1.0)
 
+seqio.MixtureRegistry.add("mt5_tydiqa_dev", tydiqa_dev, default_rate=1.0)
+
 # ----- TyDiQA GoldP Zero-Shot-----
 # This Zero-Shot setting matches the XTREME setup, where training is done on
 # the English data of TyDiQA. In the TyDiQA paper, fine-tuning was done on