Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 6 additions & 19 deletions kaievolve/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import os
import logging
from dataclasses import dataclass, field
from dataclasses import dataclass, field, fields
from pathlib import Path
from typing import Any, Dict, List, Optional, Union

Expand Down Expand Up @@ -699,24 +699,11 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config":
config_dict["llm"] = {}
config_dict["llm"]["evaluator_generation_models"] = evaluator_generation_models

# Filter out keys that don't belong to EvaluatorConfig
evaluator_config_fields = {
"timeout",
"max_retries",
"memory_limit_mb",
"cpu_limit",
"cascade_evaluation",
"cascade_thresholds",
"parallel_evaluations",
"distributed",
"use_llm_feedback",
"llm_feedback_weight",
"llm_criteria",
"llm_evaluation_prompt",
"enable_artifacts",
"max_artifact_storage",
"models",
}
# Keep only keys that are real EvaluatorConfig fields. Derived from
# the dataclass (not a hand-maintained allowlist) so a newly-added
# field — e.g. re_evaluations (#12) — loads from YAML automatically
# instead of being silently dropped.
evaluator_config_fields = {f.name for f in fields(EvaluatorConfig)}
filtered_evaluator_dict = {
k: v for k, v in evaluator_dict.items() if k in evaluator_config_fields
}
Expand Down
37 changes: 37 additions & 0 deletions tests/test_config_loading.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""Config YAML-load coverage — guards the bug where evaluator.re_evaluations
(#12) was silently dropped by a hand-maintained allowlist in load_config."""

import os
import tempfile
import unittest

from kaievolve.config import load_config


class TestEvaluatorConfigLoading(unittest.TestCase):
_LLM = "llm:\n models:\n - name: test-model\n"

def _load(self, yaml_text):
with tempfile.TemporaryDirectory() as d:
p = os.path.join(d, "c.yaml")
with open(p, "w") as f:
f.write(self._LLM + yaml_text)
return load_config(p)

def test_re_evaluations_loads_from_yaml(self):
cfg = self._load("evaluator:\n re_evaluations: 3\n cascade_evaluation: true\n")
self.assertEqual(cfg.evaluator.re_evaluations, 3) # was silently dropped before the fix

def test_default_when_absent(self):
cfg = self._load("evaluator:\n cascade_evaluation: true\n")
self.assertEqual(cfg.evaluator.re_evaluations, 1)

def test_unknown_evaluator_key_is_filtered_not_crashed(self):
# The allowlist's real purpose — drop keys that aren't EvaluatorConfig
# fields — must still hold (no TypeError on an unexpected key).
cfg = self._load("evaluator:\n re_evaluations: 2\n totally_bogus_key: 99\n")
self.assertEqual(cfg.evaluator.re_evaluations, 2)


if __name__ == "__main__":
unittest.main()
Loading