From 0ed7cc1588f933a753a700d23dfa04db2dd2e322 Mon Sep 17 00:00:00 2001 From: aktasbatuhan Date: Wed, 3 Jun 2026 17:11:03 +0100 Subject: [PATCH] fix(config): load evaluator.re_evaluations from YAML The evaluator section was filtered through a hand-maintained allowlist that omitted re_evaluations (added for #12), so it was silently dropped on load: setting evaluator.re_evaluations in YAML had no effect, the noise-aware re-evaluation never ran, and #12's A/B was invalid (both arms ran re_eval=1). Derive the allowlist from EvaluatorConfig's dataclass fields instead, so any field loads from YAML automatically and can't silently drop again. Add test_config_loading.py to cover the YAML-load path that was untested. Co-Authored-By: Claude Opus 4.8 (1M context) --- kaievolve/config.py | 25 ++++++------------------ tests/test_config_loading.py | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 tests/test_config_loading.py diff --git a/kaievolve/config.py b/kaievolve/config.py index 6815ef7..5ca3bfc 100644 --- a/kaievolve/config.py +++ b/kaievolve/config.py @@ -4,7 +4,7 @@ import os import logging -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -699,24 +699,11 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config": config_dict["llm"] = {} config_dict["llm"]["evaluator_generation_models"] = evaluator_generation_models - # Filter out keys that don't belong to EvaluatorConfig - evaluator_config_fields = { - "timeout", - "max_retries", - "memory_limit_mb", - "cpu_limit", - "cascade_evaluation", - "cascade_thresholds", - "parallel_evaluations", - "distributed", - "use_llm_feedback", - "llm_feedback_weight", - "llm_criteria", - "llm_evaluation_prompt", - "enable_artifacts", - "max_artifact_storage", - "models", - } + # Keep only keys that are real EvaluatorConfig fields. Derived from + # the dataclass (not a hand-maintained allowlist) so a newly-added + # field — e.g. re_evaluations (#12) — loads from YAML automatically + # instead of being silently dropped. + evaluator_config_fields = {f.name for f in fields(EvaluatorConfig)} filtered_evaluator_dict = { k: v for k, v in evaluator_dict.items() if k in evaluator_config_fields } diff --git a/tests/test_config_loading.py b/tests/test_config_loading.py new file mode 100644 index 0000000..c3b3d64 --- /dev/null +++ b/tests/test_config_loading.py @@ -0,0 +1,37 @@ +"""Config YAML-load coverage — guards the bug where evaluator.re_evaluations +(#12) was silently dropped by a hand-maintained allowlist in load_config.""" + +import os +import tempfile +import unittest + +from kaievolve.config import load_config + + +class TestEvaluatorConfigLoading(unittest.TestCase): + _LLM = "llm:\n models:\n - name: test-model\n" + + def _load(self, yaml_text): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "c.yaml") + with open(p, "w") as f: + f.write(self._LLM + yaml_text) + return load_config(p) + + def test_re_evaluations_loads_from_yaml(self): + cfg = self._load("evaluator:\n re_evaluations: 3\n cascade_evaluation: true\n") + self.assertEqual(cfg.evaluator.re_evaluations, 3) # was silently dropped before the fix + + def test_default_when_absent(self): + cfg = self._load("evaluator:\n cascade_evaluation: true\n") + self.assertEqual(cfg.evaluator.re_evaluations, 1) + + def test_unknown_evaluator_key_is_filtered_not_crashed(self): + # The allowlist's real purpose — drop keys that aren't EvaluatorConfig + # fields — must still hold (no TypeError on an unexpected key). + cfg = self._load("evaluator:\n re_evaluations: 2\n totally_bogus_key: 99\n") + self.assertEqual(cfg.evaluator.re_evaluations, 2) + + +if __name__ == "__main__": + unittest.main()