diff --git a/CHANGELOG.md b/CHANGELOG.md index 513840a9..f43c0e80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ All notable changes to SkillOpt are documented here. This project adheres to ## [Unreleased] ### Added +- **SkillOpt-Sleep opt-in `llm_dream`**: paraphrase-only dream variants from + the optimizer model, with deterministic template fallback on parse or + fidelity failure. Default template dreams stay byte-identical; generated + variants are train-only (thanks @bogdanbaciu21). - **SkillOpt-Sleep multi-skill fan-out and reviewed subset adoption**: each hinted skill is consolidated from its own pinned live baseline, staged as an independent proposal with per-skill gate evidence, and promoted only through diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 2576c127..f4781ac6 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -338,6 +338,7 @@ correctness signal; the validation gate still governs what ships. | `dream_rollouts` | `1` | Run each task K times → learn from the good-vs-bad contrast (contrastive reflection). | | `recall_k` | `0` | Associative recall — pull the K most-similar past tasks (from a persisted archive) into tonight's dream. | | `dream_factor` | `0` | Add N lightweight synthetic variants of each task. | +| `llm_dream` | `false` | Opt-in paraphrase generator for those variants. Templates stay the default and are used on any parse or fidelity failure. v1 is paraphrase-only: parent `reference`/`judge` are copied unchanged. | ## Results diff --git a/plugins/README.md b/plugins/README.md index b3aba6bc..318513a4 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -250,7 +250,7 @@ python -m skillopt_sleep run --project "$(pwd)" \ The JSON/YAML config under `~/.skillopt-sleep/` supports additional engine keys, including `gate_mode`, `gate_metric`, `gate_no_regression`, `dream_rollouts`, -`dream_factor`, `recall_k`, `evolve_memory`, and `evolve_skill`. These are config +`dream_factor`, `llm_dream`, `recall_k`, `evolve_memory`, and `evolve_skill`. These are config keys, not aliases for the unsupported CLI flags listed above. Shipping defaults are conservative: `gate_mode="on"`, `gate_no_regression=false`, `dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`. diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index d4a008d1..0a08e2b9 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -70,6 +70,7 @@ # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─ "dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task "dream_factor": 0, # >0 => add N synthetic variants of each task to the dream + "llm_dream": False, # opt-in paraphrase generator; templates stay the default "recall_k": 0, # >0 => recall the K most-similar past tasks into the dream "evolve_memory": True, # consolidate CLAUDE.md "evolve_skill": True, # consolidate the managed SKILL.md diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index f1d7bf02..d710beb4 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -22,7 +22,7 @@ from skillopt_sleep import evidence from skillopt_sleep.backend import Backend, CursorBackendError, build_backend from skillopt_sleep.config import DEFAULTS, SleepConfig, load_config -from skillopt_sleep.dream import dream_consolidate +from skillopt_sleep.dream import backend_generate_fn, dream_consolidate from skillopt_sleep.evidence import EvidenceLog from skillopt_sleep.harvest_sources import harvest_for_config from skillopt_sleep.memory import ensure_skill_scaffold @@ -695,7 +695,7 @@ def run_sleep_cycle( "target_backend", "target_model", "gate_mode", "gate_metric", "gate_mixed_weight", "gate_no_regression", "edit_budget", "holdout_fraction", "val_fraction", "test_fraction", - "dream_rollouts", "dream_factor", "recall_k", + "dream_rollouts", "dream_factor", "llm_dream", "recall_k", "max_tasks_per_night", "lookback_hours", "llm_mine", "evolve_skill", "evolve_memory")} cycle_config["opencode_tool_replay"] = ( @@ -859,6 +859,11 @@ def run_sleep_cycle( recall_k=recall_k, dream_rollouts=int(cfg.get("dream_rollouts", 1) or 1), dream_factor=int(cfg.get("dream_factor", 0) or 0), + llm_dream=bool(cfg.get("llm_dream", False)), + generate_fn=( + backend_generate_fn(backend) if cfg.get("llm_dream", False) else None + ), + evidence=ev, edit_budget=cfg.get("edit_budget", 4), gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), @@ -953,6 +958,11 @@ def run_sleep_cycle( recall_k=recall_k, dream_rollouts=int(cfg.get("dream_rollouts", 1) or 1), dream_factor=int(cfg.get("dream_factor", 0) or 0), + llm_dream=bool(cfg.get("llm_dream", False)), + generate_fn=( + backend_generate_fn(backend) if cfg.get("llm_dream", False) else None + ), + evidence=ev, edit_budget=cfg.get("edit_budget", 4), gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py index 9906e07d..8790f705 100644 --- a/skillopt_sleep/dream.py +++ b/skillopt_sleep/dream.py @@ -17,11 +17,13 @@ from __future__ import annotations import re -from typing import List, Optional +from typing import Callable, List, Optional from skillopt_sleep.consolidate import ConsolidationResult, consolidate from skillopt_sleep.types import TaskRecord +GenerateFn = Callable[[str], str] + # ── synthetic augmentation ("dream up" variants of today's tasks) ───────────── _WRAPPERS = [ @@ -31,28 +33,139 @@ ] -def dream_augment(real_tasks: List[TaskRecord], *, factor: int = 1) -> List[TaskRecord]: +def _template_intent(task: TaskRecord, k: int) -> str: + return _WRAPPERS[k % len(_WRAPPERS)].format(q=task.intent) + + +def _parse_paraphrases(raw: str, n: int) -> List[str]: + """Accept a JSON array of strings; drop empty / short / non-string items.""" + from skillopt_sleep.backend import _extract_json + parsed = _extract_json(raw or "", "array") + if not isinstance(parsed, list): + return [] + out: List[str] = [] + for item in parsed: + if not isinstance(item, str): + continue + text = item.strip() + if len(text) < 8: + continue + out.append(text) + if len(out) >= n: + break + return out + + +def _fidelity_ok(original: str, paraphrase: str) -> bool: + """Paraphrase-only v1: keep the parent's constraints by construction. + + We copy reference/judge unchanged, so a constraint-changing rewrite would + mislabel the variant. Refuse empty, identical, and prompt-echo strings. + Constraint perturbations are deferred to a later redesign of judge + propagation. + """ + text = (paraphrase or "").strip() + src = (original or "").strip() + if len(text) < 8 or not src: + return False + if text == src: + return False + if "Return ONLY a JSON array" in text: + return False + return True + + +def _dream_record(task: TaskRecord, k: int, intent: str, extra_tags: Optional[List[str]] = None) -> TaskRecord: + tags = list(task.tags) + ["dream"] + if extra_tags: + tags.extend(extra_tags) + return TaskRecord( + id=f"{task.id}_dream{k}", project=task.project, + intent=intent, context_excerpt=task.context_excerpt, + reference_kind=task.reference_kind, reference=task.reference, + judge=dict(task.judge), system=task.system, + tags=tags, split="train", + origin="dream", derived_from=task.id, + skill_hint=task.skill_hint, + ) + + +def dream_augment( + real_tasks: List[TaskRecord], + *, + factor: int = 1, + llm_dream: bool = False, + generate_fn: Optional[GenerateFn] = None, + evidence=None, +) -> List[TaskRecord]: """Create synthetic TRAIN variants of real tasks (origin='dream'). - A light, deterministic rephrasing. Dream tasks are training-only — they - carry split='train' and never enter the val/test slices the gate scores on. + Default path is a light, deterministic rephrasing. Dream tasks are + training-only: they carry split='train' and never enter the val/test + slices the gate scores on. + + Opt-in ``llm_dream=True`` asks ``generate_fn`` for paraphrase-only + rewrites (parent reference/judge copied unchanged). Any parse or + fidelity failure falls back to the same wrappers as the default path, + so a night can degrade but not break. Template mode (the default) is + byte-identical to the pre-llm_dream implementation. """ out: List[TaskRecord] = [] + use_llm = bool(llm_dream) and generate_fn is not None + if llm_dream and generate_fn is None and evidence is not None: + evidence.log( + "dream", "llm_dream_fallback", + reason="no_generate_fn", n_requested=max(0, factor), + ) for t in real_tasks: + parsed: List[str] = [] + if use_llm: + try: + from skillopt_sleep import prompts as prompt_registry + prompt = prompt_registry.render("llm_dream", { + "__INTENT__": t.intent, + "__N__": str(max(0, factor)), + "__CONTEXT__": (t.context_excerpt or "")[:400], + }) + parsed = _parse_paraphrases(generate_fn(prompt), max(0, factor)) + except Exception: + parsed = [] + n_ok = 0 for k in range(max(0, factor)): - w = _WRAPPERS[k % len(_WRAPPERS)] - out.append(TaskRecord( - id=f"{t.id}_dream{k}", project=t.project, - intent=w.format(q=t.intent), context_excerpt=t.context_excerpt, - reference_kind=t.reference_kind, reference=t.reference, - judge=dict(t.judge), system=t.system, - tags=list(t.tags) + ["dream"], split="train", - origin="dream", derived_from=t.id, - skill_hint=t.skill_hint, - )) + extra: Optional[List[str]] = None + if use_llm and k < len(parsed) and _fidelity_ok(t.intent, parsed[k]): + intent = parsed[k] + extra = ["llm_dream"] + n_ok += 1 + else: + intent = _template_intent(t, k) + out.append(_dream_record(t, k, intent, extra)) + if use_llm and n_ok < max(0, factor) and evidence is not None: + evidence.log( + "dream", "llm_dream_fallback", + task_id=t.id, + n_fallback=max(0, factor) - n_ok, + n_requested=max(0, factor), + ) return out +def backend_generate_fn(backend) -> GenerateFn: + """Adapter: reuse Backend.attempt so every backend can paraphrase. + + The probe task is never added to the training pool. + """ + def generate(prompt: str) -> str: + probe = TaskRecord( + id="__llm_dream_probe__", + project="", + intent=prompt, + reference_kind="none", + ) + return backend.attempt(probe, skill="", memory="") + return generate + + # ── associative recall (experience replay of similar past tasks) ────────────── def _tokens(text: str) -> set: @@ -134,6 +247,9 @@ def dream_consolidate( evolve_skill: bool = True, evolve_memory: bool = True, night: int = 1, + llm_dream: bool = False, + generate_fn: Optional[GenerateFn] = None, + evidence=None, ) -> ConsolidationResult: """Recall similar past experience + dream synthetic variants, then run one gated consolidation epoch over the enlarged training pool. @@ -157,7 +273,13 @@ def dream_consolidate( ) if dream_factor > 0: seed = [t for t in enlarged if t.split == "train" and t.origin != "dream"] - enlarged += dream_augment(seed, factor=dream_factor) + enlarged += dream_augment( + seed, + factor=dream_factor, + llm_dream=llm_dream, + generate_fn=generate_fn, + evidence=evidence, + ) return consolidate( backend, enlarged, skill, memory, edit_budget=edit_budget, gate_metric=gate_metric, diff --git a/skillopt_sleep/prompts.py b/skillopt_sleep/prompts.py index 9d896e15..c5edc467 100644 --- a/skillopt_sleep/prompts.py +++ b/skillopt_sleep/prompts.py @@ -117,6 +117,21 @@ "# Recurring failures\n__FAILURES__" ) +_LLM_DREAM = """You rewrite one existing task as a paraphrase-only variant. + +Do NOT change the task's constraints, success criteria, required answer, tools, +or output format. Do NOT invent new requirements. Keep the same meaning. + +Original intent: +__INTENT__ + +Optional context: +__CONTEXT__ + +Return ONLY a JSON array of exactly __N__ distinct paraphrase strings. +Example: ["please handle this request: ...", "for the daily report: ..."] +""" + # name -> {text, stage, role, description, placeholders} DEFAULTS: Dict[str, Dict] = { "miner": { @@ -150,6 +165,13 @@ "__CRITERIA__", "__PREFS__", "__FAILURES__", ], }, + "llm_dream": { + "text": _LLM_DREAM, + "stage": "dream", + "role": "optimizer", + "description": "Paraphrase-only dream variants; parent judge/reference stay valid.", + "placeholders": ["__INTENT__", "__N__", "__CONTEXT__"], + }, } diff --git a/tests/test_llm_dream.py b/tests/test_llm_dream.py new file mode 100644 index 00000000..7bba2f0c --- /dev/null +++ b/tests/test_llm_dream.py @@ -0,0 +1,248 @@ +"""Opt-in llm_dream: paraphrase-only, train-only, deterministic fallback.""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest + +from skillopt_sleep.config import DEFAULTS, load_config +from skillopt_sleep.cycle import run_sleep_cycle +from skillopt_sleep.dream import ( + _WRAPPERS, + _fidelity_ok, + _parse_paraphrases, + dream_augment, + dream_consolidate, +) +from skillopt_sleep.types import TaskRecord + + +def _task(tid: str = "t1", intent: str = "add form validation to the signup page") -> TaskRecord: + return TaskRecord( + id=tid, + project="/p", + intent=intent, + reference_kind="exact", + reference="use the shared validator", + judge={"checks": [{"op": "contains", "arg": "validator"}]}, + split="train", + origin="real", + skill_hint="forms", + tags=["rule:wrap-answer"], + ) + + +class TestTemplateDefaultUnchanged(unittest.TestCase): + def test_default_matches_hardcoded_wrappers(self): + src = _task() + got = dream_augment([src], factor=3) + self.assertEqual(len(got), 3) + for k, dream in enumerate(got): + self.assertEqual(dream.intent, _WRAPPERS[k].format(q=src.intent)) + self.assertEqual(dream.split, "train") + self.assertEqual(dream.origin, "dream") + self.assertEqual(dream.derived_from, src.id) + self.assertEqual(dream.reference, src.reference) + self.assertEqual(dream.judge, src.judge) + self.assertEqual(dream.tags, src.tags + ["dream"]) + self.assertNotIn("llm_dream", dream.tags) + self.assertEqual(dream.skill_hint, "forms") + + def test_llm_dream_false_ignores_generator(self): + src = _task() + calls = [] + + def gen(prompt: str) -> str: + calls.append(prompt) + return json.dumps(["totally different paraphrase of the request"]) + + got = dream_augment([src], factor=1, llm_dream=False, generate_fn=gen) + self.assertEqual(calls, []) + self.assertEqual(got[0].intent, _WRAPPERS[0].format(q=src.intent)) + + +class TestParseAndFidelity(unittest.TestCase): + def test_parse_json_array(self): + raw = 'Sure.\n["please add signup validation", "handle signup form checks"]\n' + self.assertEqual( + _parse_paraphrases(raw, 2), + ["please add signup validation", "handle signup form checks"], + ) + + def test_parse_rejects_garbage(self): + self.assertEqual(_parse_paraphrases("not json", 2), []) + self.assertEqual(_parse_paraphrases('{"intent": "x"}', 1), []) + + def test_fidelity_rejects_identical_and_prompt_echo(self): + src = "add form validation to the signup page" + self.assertFalse(_fidelity_ok(src, src)) + self.assertFalse(_fidelity_ok(src, "short")) + self.assertFalse(_fidelity_ok(src, "Return ONLY a JSON array of junk")) + self.assertTrue(_fidelity_ok(src, "please add validation on the signup form")) + + +class TestLlmDreamPath(unittest.TestCase): + def test_valid_paraphrases_are_used(self): + src = _task() + + def gen(_prompt: str) -> str: + return json.dumps([ + "please add validation on the signup form", + "handle signup-page form checks", + ]) + + got = dream_augment([src], factor=2, llm_dream=True, generate_fn=gen) + self.assertEqual(got[0].intent, "please add validation on the signup form") + self.assertEqual(got[1].intent, "handle signup-page form checks") + for dream in got: + self.assertEqual(dream.split, "train") + self.assertEqual(dream.origin, "dream") + self.assertIn("llm_dream", dream.tags) + self.assertEqual(dream.reference, src.reference) + self.assertEqual(dream.judge, src.judge) + + def test_parse_failure_falls_back_deterministically(self): + src = _task() + events = [] + + class _Ev: + def log(self, stage, event, **data): + events.append((stage, event, data)) + + def gen(_prompt: str) -> str: + return "I cannot comply" + + a = dream_augment([src], factor=2, llm_dream=True, generate_fn=gen, evidence=_Ev()) + b = dream_augment([src], factor=2, llm_dream=True, generate_fn=gen, evidence=_Ev()) + self.assertEqual([d.intent for d in a], [d.intent for d in b]) + self.assertEqual(a[0].intent, _WRAPPERS[0].format(q=src.intent)) + self.assertEqual(a[1].intent, _WRAPPERS[1].format(q=src.intent)) + self.assertNotIn("llm_dream", a[0].tags) + self.assertTrue(any(ev[1] == "llm_dream_fallback" for ev in events)) + + def test_partial_parse_fills_rest_from_templates(self): + src = _task() + + def gen(_prompt: str) -> str: + return json.dumps(["please add validation on the signup form"]) + + got = dream_augment([src], factor=2, llm_dream=True, generate_fn=gen) + self.assertEqual(got[0].intent, "please add validation on the signup form") + self.assertEqual(got[1].intent, _WRAPPERS[1].format(q=src.intent)) + self.assertIn("llm_dream", got[0].tags) + self.assertNotIn("llm_dream", got[1].tags) + + def test_generator_exception_falls_back(self): + src = _task() + + def gen(_prompt: str) -> str: + raise RuntimeError("backend down") + + got = dream_augment([src], factor=1, llm_dream=True, generate_fn=gen) + self.assertEqual(got[0].intent, _WRAPPERS[0].format(q=src.intent)) + + def test_llm_dream_without_generator_uses_templates(self): + src = _task() + got = dream_augment([src], factor=1, llm_dream=True, generate_fn=None) + self.assertEqual(got[0].intent, _WRAPPERS[0].format(q=src.intent)) + + +class TestSplitHygiene(unittest.TestCase): + def test_llm_dreams_never_leave_train(self): + val = _task("val1") + val.split = "val" + test = _task("test1") + test.split = "test" + + def gen(_prompt: str) -> str: + return json.dumps(["please add validation on the signup form"]) + + dreamed = dream_augment( + [val, test], factor=1, llm_dream=True, generate_fn=gen, + ) + self.assertEqual({d.split for d in dreamed}, {"train"}) + self.assertEqual({d.origin for d in dreamed}, {"dream"}) + + def test_dream_consolidate_keeps_val_clean(self): + from skillopt_sleep.backend import MockBackend + + train = _task("tr") + val = _task("va", intent="score the holdout form task") + val.split = "val" + val.reference = "holdout-answer" + calls = [] + + def gen(prompt: str) -> str: + calls.append(prompt) + return json.dumps(["please add validation on the signup form"]) + + res = dream_consolidate( + MockBackend(), + [train, val], + skill="", + memory="", + dream_factor=1, + llm_dream=True, + generate_fn=gen, + gate_mode="off", + ) + self.assertIsNotNone(res) + self.assertTrue(calls) + # The generator is only asked to rewrite train tasks (val is not a seed). + self.assertTrue(any("add form validation" in p for p in calls)) + self.assertFalse(any("score the holdout" in p for p in calls)) + + +class TestConfigDefaultOff(unittest.TestCase): + def test_default_is_false(self): + self.assertFalse(DEFAULTS["llm_dream"]) + cfg = load_config() + self.assertFalse(cfg.get("llm_dream")) + + def test_cycle_default_does_not_call_generator(self): + src = _task() + src.tags = ["rule:wrap-answer"] + with tempfile.TemporaryDirectory() as proj, tempfile.TemporaryDirectory() as home: + cfg = load_config( + invoked_project=proj, + projects="invoked", + backend="mock", + claude_home=os.path.join(home, ".claude"), + dream_factor=2, + auto_adopt=False, + evidence_log=False, + ) + outcome = run_sleep_cycle(cfg, seed_tasks=[src]) + self.assertIsNotNone(outcome) + + +class TestDiversityAccounting(unittest.TestCase): + def test_llm_intents_are_more_distinct_than_templates_on_fixture(self): + src = _task() + templates = dream_augment([src], factor=3) + paraphrases = [ + "please add validation on the signup form", + "signup page needs the shared form checks", + "apply the validator before accepting a signup", + ] + + def gen(_prompt: str) -> str: + return json.dumps(paraphrases) + + llm = dream_augment([src], factor=3, llm_dream=True, generate_fn=gen) + + def distinct_1(texts): + toks = [] + for text in texts: + toks.extend(w for w in text.lower().split() if len(w) > 2) + return (len(set(toks)) / len(toks)) if toks else 0.0 + + self.assertGreater( + distinct_1([d.intent for d in llm]), + distinct_1([d.intent for d in templates]), + ) + + +if __name__ == "__main__": + unittest.main()