Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 108 additions & 0 deletions examples/noisy_optimization/evaluator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
"""Stochastic evaluator for the noisy-optimization benchmark (issue #12).

The candidate's construct() returns a vector whose TRUE quality is a
deterministic concave function maximized at a hidden target. But evaluate()
reports that quality with FRESH additive Gaussian noise on every call, so
repeated evaluations of the SAME program differ — exactly the regime where one
score is a noisy sample and selection can be fooled by a lucky draw.

The noise uses an entropy-seeded RNG created per call, so it stays genuinely
random even if the framework seeds the global RNGs for reproducibility.

true_evaluate() returns the NOISELESS quality. It is used only by the benchmark
to judge how good the selected program really is — never during a run.
"""

import json
import subprocess
import sys
import time

import numpy as np

DIM = 8
TARGET = np.array([0.5, -0.3, 0.8, -0.6, 0.2, -0.9, 0.4, -0.1])
SIGMA = 0.10 # noise std on the reported score
BOUND = 1.5


# Run construct() in a CLEAN subprocess that imports only the candidate file, so
# it cannot introspect this evaluator's memory (e.g. read the hidden TARGET off
# the call stack) — the candidate must actually optimize, not reward-hack.
_RUNNER = (
"import importlib.util,json,sys;"
"s=importlib.util.spec_from_file_location('c',sys.argv[1]);"
"m=importlib.util.module_from_spec(s);s.loader.exec_module(m);"
"print(json.dumps([float(x) for x in m.construct()]))"
)


def _load_vector(program_path: str) -> np.ndarray:
out = subprocess.run(
[sys.executable, "-c", _RUNNER, program_path],
capture_output=True,
text=True,
timeout=15,
)
if out.returncode != 0:
raise RuntimeError((out.stderr.strip() or "construct() failed")[:200])
arr = np.array(json.loads(out.stdout.strip().splitlines()[-1]), dtype=float).flatten()
if arr.shape[0] != DIM:
raise ValueError(f"construct() must return {DIM} numbers, got {arr.shape[0]}")
return np.clip(arr, -BOUND, BOUND)


def _true_score(v: np.ndarray) -> float:
"""Noiseless quality in [0, 1]; 1.0 exactly at the hidden target."""
return float(max(0.0, 1.0 - float(np.mean((v - TARGET) ** 2))))


def _noisy_score(v: np.ndarray) -> float:
# Fresh entropy-seeded RNG each call => genuinely different noise every
# evaluation, immune to any global seeding done by the framework.
rng = np.random.default_rng()
return float(np.clip(_true_score(v) + rng.normal(0.0, SIGMA), 0.0, 1.0))


def _result(program_path: str, stage: float) -> dict:
t0 = time.time()
try:
v = _load_vector(program_path)
except Exception as e:
return {
"runs_successfully": 0.0,
"combined_score": 0.0,
"stage": stage,
"stderr": f"{type(e).__name__}: {e}",
"wall_seconds": time.time() - t0,
}
return {
"runs_successfully": 1.0,
"combined_score": _noisy_score(v), # NOISY — a single sample
"stage": stage,
"wall_seconds": time.time() - t0,
}


def evaluate_stage1(program_path: str) -> dict:
return _result(program_path, 1.0)


def evaluate(program_path: str) -> dict:
return _result(program_path, 2.0)


evaluate_stage2 = evaluate


def true_evaluate(program_path: str) -> float:
"""Noiseless quality of the program — for benchmark judging only."""
return _true_score(_load_vector(program_path))


if __name__ == "__main__":
import sys

p = sys.argv[1] if len(sys.argv) > 1 else "initial_program.py"
print(f"true_score = {true_evaluate(p):.4f}")
print("noisy samples =", [round(evaluate(p)["combined_score"], 4) for _ in range(6)])
27 changes: 27 additions & 0 deletions examples/noisy_optimization/initial_program.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
"""Noisy optimization benchmark (issue #12).

Evolve construct() to return an 8-number vector as close as possible to the
evaluator's hidden target. The catch: the score you receive is a NOISY sample of
the program's true quality — repeated evaluations of the same vector differ — so
a single high score may just be luck. This is a stochastic evaluator, used to
validate noise-aware fitness (tracking score uncertainty + re-evaluating
promising-but-uncertain candidates).
"""


# EVOLVE-BLOCK-START
def construct():
"""Return a list of 8 floats, each roughly in [-1.5, 1.5].

The combined_score reported back is higher when the vector is closer to the
evaluator's fixed hidden target, but it is measured with additive noise.
Move components in whatever direction the score trend suggests.
"""
return [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]


# EVOLVE-BLOCK-END


if __name__ == "__main__":
print(construct())
8 changes: 8 additions & 0 deletions kaievolve/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,14 @@ class EvaluatorConfig:
cascade_evaluation: bool = True
cascade_thresholds: List[float] = field(default_factory=lambda: [0.5, 0.75, 0.9])

# Noise-aware fitness (issue #12). For STOCHASTIC evaluators, a single
# evaluation is a noisy sample of a program's true quality, so selection can
# be fooled by a lucky draw. When >1, each program is evaluated this many
# times and the numeric metrics (incl. combined_score) are averaged, giving
# selection a less-noisy estimate (variance shrinks ~1/n) at n x the eval
# cost. 1 (default) = current behavior; leave at 1 for deterministic tasks.
re_evaluations: int = 1

# Parallel evaluation
parallel_evaluations: int = 1
distributed: bool = False
Expand Down
53 changes: 53 additions & 0 deletions kaievolve/evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,27 @@ async def evaluate_program(
# Process the result based on type
eval_result = self._process_evaluation_result(result)

# Noise-aware fitness (issue #12): for stochastic evaluators,
# average several independent evaluations so the score used for
# selection is a less-noisy estimate of true quality. re_evaluations
# == 1 (default) is a no-op. Only re-evaluate a program that ran
# successfully the first time (don't pay to re-confirm a failure).
re_evals = getattr(self.config, "re_evaluations", 1) or 1
if (
re_evals > 1
and not eval_result.metrics.get("error")
and eval_result.metrics.get("combined_score") is not None
and eval_result.metrics.get("runs_successfully", 1.0)
):
samples = [dict(eval_result.metrics)]
for _ in range(re_evals - 1):
if self.config.cascade_evaluation:
r = await self._cascade_evaluate(temp_file_path)
else:
r = await self._direct_evaluate(temp_file_path)
samples.append(self._process_evaluation_result(r).metrics)
eval_result.metrics = self._average_metrics(samples)

# Check if this was a timeout and capture artifacts if enabled
if artifacts_enabled and program_id and eval_result.metrics.get("timeout") is True:
if program_id not in self._pending_artifacts:
Expand Down Expand Up @@ -429,6 +450,38 @@ async def evaluate_program(
)
return {"error": 0.0}

def _average_metrics(self, samples: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Average numeric metrics across repeated evaluations (issue #12).

A metric is averaged only when it is numeric (and not a bool) in *every*
sample; otherwise the last sample's value is kept (e.g. ``stderr``,
``timeout``). Records ``combined_score_std`` and ``n_evals`` so the
uncertainty is visible downstream.
"""
import statistics

keys = set().union(*(s.keys() for s in samples))
out: Dict[str, Any] = {}
for k in keys:
vals = [
s[k]
for s in samples
if isinstance(s.get(k), (int, float)) and not isinstance(s.get(k), bool)
]
if len(vals) == len(samples) and vals:
out[k] = sum(vals) / len(vals)
else:
out[k] = samples[-1].get(k)
scores = [
s.get("combined_score")
for s in samples
if isinstance(s.get("combined_score"), (int, float))
]
if len(scores) > 1:
out["combined_score_std"] = statistics.pstdev(scores)
out["n_evals"] = float(len(scores))
return out

def _process_evaluation_result(self, result: Any) -> EvaluationResult:
"""
Process evaluation result to handle both dict and EvaluationResult returns
Expand Down
100 changes: 100 additions & 0 deletions tests/test_noise_aware.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
"""Tests for noise-aware fitness: averaging repeated evaluations (issue #12)."""

import unittest

from kaievolve.config import EvaluatorConfig
from kaievolve.evaluator import Evaluator


# _average_metrics doesn't touch instance state, so we can call it unbound.
_avg = Evaluator._average_metrics.__get__(object())


class TestAverageMetrics(unittest.TestCase):
def test_numeric_metrics_averaged(self):
out = _avg(
[
{"combined_score": 0.4, "runs_successfully": 1.0, "stage": 2.0},
{"combined_score": 0.6, "runs_successfully": 1.0, "stage": 2.0},
]
)
self.assertAlmostEqual(out["combined_score"], 0.5)
self.assertAlmostEqual(out["runs_successfully"], 1.0)
self.assertEqual(out["stage"], 2.0)

def test_records_uncertainty(self):
out = _avg([{"combined_score": 0.2}, {"combined_score": 0.8}, {"combined_score": 0.5}])
self.assertAlmostEqual(out["combined_score"], 0.5)
self.assertEqual(out["n_evals"], 3.0)
self.assertGreater(out["combined_score_std"], 0.0)

def test_non_numeric_takes_last(self):
out = _avg(
[
{"combined_score": 0.5, "stderr": "first"},
{"combined_score": 0.5, "stderr": "last"},
]
)
self.assertEqual(out["stderr"], "last")

def test_bool_not_averaged(self):
# timeout is a bool flag, not a metric to average.
out = _avg(
[{"combined_score": 0.5, "timeout": True}, {"combined_score": 0.5, "timeout": True}]
)
self.assertIs(out["timeout"], True)

def test_partial_numeric_takes_last(self):
# raw_C present in only one sample -> not averaged, last value kept.
out = _avg([{"combined_score": 0.4}, {"combined_score": 0.6, "raw_C": 1.7}])
self.assertAlmostEqual(out["combined_score"], 0.5)
self.assertEqual(out["raw_C"], 1.7)

def test_averaging_shrinks_spread(self):
# Mean of noisy samples is closer to the true value than the worst sample.
true = 0.7
samples = [{"combined_score": s} for s in (0.55, 0.85, 0.6, 0.8)]
out = _avg(samples)
self.assertLess(abs(out["combined_score"] - true), 0.15)


class TestConfigDefault(unittest.TestCase):
def test_default_is_one(self):
self.assertEqual(EvaluatorConfig().re_evaluations, 1)


class TestNoisyTaskEvaluator(unittest.TestCase):
"""The benchmark task itself must be genuinely stochastic with a usable
noiseless true_evaluate (the prerequisite for validating #12)."""

def _load(self):
import importlib.util
import os

path = os.path.join(
os.path.dirname(__file__),
"..",
"examples",
"noisy_optimization",
"evaluator.py",
)
spec = importlib.util.spec_from_file_location("noisy_eval", path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod, os.path.join(os.path.dirname(path), "initial_program.py")

def test_repeated_evals_differ(self):
mod, prog = self._load()
samples = [mod.evaluate(prog)["combined_score"] for _ in range(8)]
self.assertGreater(len(set(samples)), 1) # genuinely noisy

def test_true_evaluate_is_deterministic_and_bounded(self):
mod, prog = self._load()
a = mod.true_evaluate(prog)
b = mod.true_evaluate(prog)
self.assertEqual(a, b) # noiseless
self.assertTrue(0.0 <= a <= 1.0)


if __name__ == "__main__":
unittest.main()
Loading