diff --git a/examples/noisy_optimization/evaluator.py b/examples/noisy_optimization/evaluator.py new file mode 100644 index 0000000..a005f42 --- /dev/null +++ b/examples/noisy_optimization/evaluator.py @@ -0,0 +1,108 @@ +"""Stochastic evaluator for the noisy-optimization benchmark (issue #12). + +The candidate's construct() returns a vector whose TRUE quality is a +deterministic concave function maximized at a hidden target. But evaluate() +reports that quality with FRESH additive Gaussian noise on every call, so +repeated evaluations of the SAME program differ — exactly the regime where one +score is a noisy sample and selection can be fooled by a lucky draw. + +The noise uses an entropy-seeded RNG created per call, so it stays genuinely +random even if the framework seeds the global RNGs for reproducibility. + +true_evaluate() returns the NOISELESS quality. It is used only by the benchmark +to judge how good the selected program really is — never during a run. +""" + +import json +import subprocess +import sys +import time + +import numpy as np + +DIM = 8 +TARGET = np.array([0.5, -0.3, 0.8, -0.6, 0.2, -0.9, 0.4, -0.1]) +SIGMA = 0.10 # noise std on the reported score +BOUND = 1.5 + + +# Run construct() in a CLEAN subprocess that imports only the candidate file, so +# it cannot introspect this evaluator's memory (e.g. read the hidden TARGET off +# the call stack) — the candidate must actually optimize, not reward-hack. +_RUNNER = ( + "import importlib.util,json,sys;" + "s=importlib.util.spec_from_file_location('c',sys.argv[1]);" + "m=importlib.util.module_from_spec(s);s.loader.exec_module(m);" + "print(json.dumps([float(x) for x in m.construct()]))" +) + + +def _load_vector(program_path: str) -> np.ndarray: + out = subprocess.run( + [sys.executable, "-c", _RUNNER, program_path], + capture_output=True, + text=True, + timeout=15, + ) + if out.returncode != 0: + raise RuntimeError((out.stderr.strip() or "construct() failed")[:200]) + arr = np.array(json.loads(out.stdout.strip().splitlines()[-1]), dtype=float).flatten() + if arr.shape[0] != DIM: + raise ValueError(f"construct() must return {DIM} numbers, got {arr.shape[0]}") + return np.clip(arr, -BOUND, BOUND) + + +def _true_score(v: np.ndarray) -> float: + """Noiseless quality in [0, 1]; 1.0 exactly at the hidden target.""" + return float(max(0.0, 1.0 - float(np.mean((v - TARGET) ** 2)))) + + +def _noisy_score(v: np.ndarray) -> float: + # Fresh entropy-seeded RNG each call => genuinely different noise every + # evaluation, immune to any global seeding done by the framework. + rng = np.random.default_rng() + return float(np.clip(_true_score(v) + rng.normal(0.0, SIGMA), 0.0, 1.0)) + + +def _result(program_path: str, stage: float) -> dict: + t0 = time.time() + try: + v = _load_vector(program_path) + except Exception as e: + return { + "runs_successfully": 0.0, + "combined_score": 0.0, + "stage": stage, + "stderr": f"{type(e).__name__}: {e}", + "wall_seconds": time.time() - t0, + } + return { + "runs_successfully": 1.0, + "combined_score": _noisy_score(v), # NOISY — a single sample + "stage": stage, + "wall_seconds": time.time() - t0, + } + + +def evaluate_stage1(program_path: str) -> dict: + return _result(program_path, 1.0) + + +def evaluate(program_path: str) -> dict: + return _result(program_path, 2.0) + + +evaluate_stage2 = evaluate + + +def true_evaluate(program_path: str) -> float: + """Noiseless quality of the program — for benchmark judging only.""" + return _true_score(_load_vector(program_path)) + + +if __name__ == "__main__": + import sys + + p = sys.argv[1] if len(sys.argv) > 1 else "initial_program.py" + print(f"true_score = {true_evaluate(p):.4f}") + print("noisy samples =", [round(evaluate(p)["combined_score"], 4) for _ in range(6)]) diff --git a/examples/noisy_optimization/initial_program.py b/examples/noisy_optimization/initial_program.py new file mode 100644 index 0000000..5b4b523 --- /dev/null +++ b/examples/noisy_optimization/initial_program.py @@ -0,0 +1,27 @@ +"""Noisy optimization benchmark (issue #12). + +Evolve construct() to return an 8-number vector as close as possible to the +evaluator's hidden target. The catch: the score you receive is a NOISY sample of +the program's true quality — repeated evaluations of the same vector differ — so +a single high score may just be luck. This is a stochastic evaluator, used to +validate noise-aware fitness (tracking score uncertainty + re-evaluating +promising-but-uncertain candidates). +""" + + +# EVOLVE-BLOCK-START +def construct(): + """Return a list of 8 floats, each roughly in [-1.5, 1.5]. + + The combined_score reported back is higher when the vector is closer to the + evaluator's fixed hidden target, but it is measured with additive noise. + Move components in whatever direction the score trend suggests. + """ + return [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + + +# EVOLVE-BLOCK-END + + +if __name__ == "__main__": + print(construct()) diff --git a/kaievolve/config.py b/kaievolve/config.py index b47320f..6815ef7 100644 --- a/kaievolve/config.py +++ b/kaievolve/config.py @@ -368,6 +368,14 @@ class EvaluatorConfig: cascade_evaluation: bool = True cascade_thresholds: List[float] = field(default_factory=lambda: [0.5, 0.75, 0.9]) + # Noise-aware fitness (issue #12). For STOCHASTIC evaluators, a single + # evaluation is a noisy sample of a program's true quality, so selection can + # be fooled by a lucky draw. When >1, each program is evaluated this many + # times and the numeric metrics (incl. combined_score) are averaged, giving + # selection a less-noisy estimate (variance shrinks ~1/n) at n x the eval + # cost. 1 (default) = current behavior; leave at 1 for deterministic tasks. + re_evaluations: int = 1 + # Parallel evaluation parallel_evaluations: int = 1 distributed: bool = False diff --git a/kaievolve/evaluator.py b/kaievolve/evaluator.py index 40c948b..c02236b 100644 --- a/kaievolve/evaluator.py +++ b/kaievolve/evaluator.py @@ -312,6 +312,27 @@ async def evaluate_program( # Process the result based on type eval_result = self._process_evaluation_result(result) + # Noise-aware fitness (issue #12): for stochastic evaluators, + # average several independent evaluations so the score used for + # selection is a less-noisy estimate of true quality. re_evaluations + # == 1 (default) is a no-op. Only re-evaluate a program that ran + # successfully the first time (don't pay to re-confirm a failure). + re_evals = getattr(self.config, "re_evaluations", 1) or 1 + if ( + re_evals > 1 + and not eval_result.metrics.get("error") + and eval_result.metrics.get("combined_score") is not None + and eval_result.metrics.get("runs_successfully", 1.0) + ): + samples = [dict(eval_result.metrics)] + for _ in range(re_evals - 1): + if self.config.cascade_evaluation: + r = await self._cascade_evaluate(temp_file_path) + else: + r = await self._direct_evaluate(temp_file_path) + samples.append(self._process_evaluation_result(r).metrics) + eval_result.metrics = self._average_metrics(samples) + # Check if this was a timeout and capture artifacts if enabled if artifacts_enabled and program_id and eval_result.metrics.get("timeout") is True: if program_id not in self._pending_artifacts: @@ -429,6 +450,38 @@ async def evaluate_program( ) return {"error": 0.0} + def _average_metrics(self, samples: List[Dict[str, Any]]) -> Dict[str, Any]: + """Average numeric metrics across repeated evaluations (issue #12). + + A metric is averaged only when it is numeric (and not a bool) in *every* + sample; otherwise the last sample's value is kept (e.g. ``stderr``, + ``timeout``). Records ``combined_score_std`` and ``n_evals`` so the + uncertainty is visible downstream. + """ + import statistics + + keys = set().union(*(s.keys() for s in samples)) + out: Dict[str, Any] = {} + for k in keys: + vals = [ + s[k] + for s in samples + if isinstance(s.get(k), (int, float)) and not isinstance(s.get(k), bool) + ] + if len(vals) == len(samples) and vals: + out[k] = sum(vals) / len(vals) + else: + out[k] = samples[-1].get(k) + scores = [ + s.get("combined_score") + for s in samples + if isinstance(s.get("combined_score"), (int, float)) + ] + if len(scores) > 1: + out["combined_score_std"] = statistics.pstdev(scores) + out["n_evals"] = float(len(scores)) + return out + def _process_evaluation_result(self, result: Any) -> EvaluationResult: """ Process evaluation result to handle both dict and EvaluationResult returns diff --git a/tests/test_noise_aware.py b/tests/test_noise_aware.py new file mode 100644 index 0000000..eb04c1e --- /dev/null +++ b/tests/test_noise_aware.py @@ -0,0 +1,100 @@ +"""Tests for noise-aware fitness: averaging repeated evaluations (issue #12).""" + +import unittest + +from kaievolve.config import EvaluatorConfig +from kaievolve.evaluator import Evaluator + + +# _average_metrics doesn't touch instance state, so we can call it unbound. +_avg = Evaluator._average_metrics.__get__(object()) + + +class TestAverageMetrics(unittest.TestCase): + def test_numeric_metrics_averaged(self): + out = _avg( + [ + {"combined_score": 0.4, "runs_successfully": 1.0, "stage": 2.0}, + {"combined_score": 0.6, "runs_successfully": 1.0, "stage": 2.0}, + ] + ) + self.assertAlmostEqual(out["combined_score"], 0.5) + self.assertAlmostEqual(out["runs_successfully"], 1.0) + self.assertEqual(out["stage"], 2.0) + + def test_records_uncertainty(self): + out = _avg([{"combined_score": 0.2}, {"combined_score": 0.8}, {"combined_score": 0.5}]) + self.assertAlmostEqual(out["combined_score"], 0.5) + self.assertEqual(out["n_evals"], 3.0) + self.assertGreater(out["combined_score_std"], 0.0) + + def test_non_numeric_takes_last(self): + out = _avg( + [ + {"combined_score": 0.5, "stderr": "first"}, + {"combined_score": 0.5, "stderr": "last"}, + ] + ) + self.assertEqual(out["stderr"], "last") + + def test_bool_not_averaged(self): + # timeout is a bool flag, not a metric to average. + out = _avg( + [{"combined_score": 0.5, "timeout": True}, {"combined_score": 0.5, "timeout": True}] + ) + self.assertIs(out["timeout"], True) + + def test_partial_numeric_takes_last(self): + # raw_C present in only one sample -> not averaged, last value kept. + out = _avg([{"combined_score": 0.4}, {"combined_score": 0.6, "raw_C": 1.7}]) + self.assertAlmostEqual(out["combined_score"], 0.5) + self.assertEqual(out["raw_C"], 1.7) + + def test_averaging_shrinks_spread(self): + # Mean of noisy samples is closer to the true value than the worst sample. + true = 0.7 + samples = [{"combined_score": s} for s in (0.55, 0.85, 0.6, 0.8)] + out = _avg(samples) + self.assertLess(abs(out["combined_score"] - true), 0.15) + + +class TestConfigDefault(unittest.TestCase): + def test_default_is_one(self): + self.assertEqual(EvaluatorConfig().re_evaluations, 1) + + +class TestNoisyTaskEvaluator(unittest.TestCase): + """The benchmark task itself must be genuinely stochastic with a usable + noiseless true_evaluate (the prerequisite for validating #12).""" + + def _load(self): + import importlib.util + import os + + path = os.path.join( + os.path.dirname(__file__), + "..", + "examples", + "noisy_optimization", + "evaluator.py", + ) + spec = importlib.util.spec_from_file_location("noisy_eval", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod, os.path.join(os.path.dirname(path), "initial_program.py") + + def test_repeated_evals_differ(self): + mod, prog = self._load() + samples = [mod.evaluate(prog)["combined_score"] for _ in range(8)] + self.assertGreater(len(set(samples)), 1) # genuinely noisy + + def test_true_evaluate_is_deterministic_and_bounded(self): + mod, prog = self._load() + a = mod.true_evaluate(prog) + b = mod.true_evaluate(prog) + self.assertEqual(a, b) # noiseless + self.assertTrue(0.0 <= a <= 1.0) + + +if __name__ == "__main__": + unittest.main()