diff --git a/src/levanter/eval_harness.py b/src/levanter/eval_harness.py
index bfa75f46a..97b6a15d6 100644
--- a/src/levanter/eval_harness.py
+++ b/src/levanter/eval_harness.py
@@ -365,7 +365,7 @@ def _get_task_and_rename(self, manager, our_name, task: dict | str):
         print("this_task type after pop: ", type(this_task))
         # hacky, but this allows us to run multiple instances of the same task with different fewshot settings
         
-        if not isinstance(task, dict):
+        if not isinstance(this_task, dict):
             # if it is a task, change the config to rename
             this_task.config.task = our_name
         else: